clear;
datasets					= { 'sept_07_2011' }; %{ 'may_03_2011','jan_25_2011','apr_09_2010','feb_26_2010','SignStreamDatabases_no_compounds','jan_27_2010','apr_30_2010' };

addpath(						'/research/asl-data3/ashwin/asl/code/hand_detector',	...
								'/research/asl-data4/ashwin/asl/software/video_compressor/vid_reader' );
xml_files_base_directory	= 	'/research/asl-data4/ASL_lexicon_project/asl-data2/quicktime/annotate/SignStream_annotations/annotations_with_compounds/processing_code/';
gloss_translation_file 		= 	'../jessica_glossing_conventions/GlossingConventions3-2-11.csv';
signers_and_sessions_list;
	
clear( 'gloss_infos', 'gloss_info' );
all_gloss_dir								= '/research/asl-data4/ASL_lexicon_project/asl-data2/quicktime/annotate/all_sign_glosses/';
[ corpus_gloss_id, corpus_gloss ]			= textread( [ all_gloss_dir, 'all_signs.csv' ], '%s%s', 'delimiter', ',' );
corpus_gloss								= lower( corpus_gloss );
corpus_gloss_processed						= { };
corpus_gloss_is_loan_sign					= [ ];
for corpusi									= 1 : length( corpus_gloss );
	cur_gloss								= corpus_gloss{ corpusi };
	cur_gloss( strfind( cur_gloss,'-' ) ) 	= [ ];				cur_gloss( strfind( cur_gloss,'_' ) ) 	= [ ];
	corpus_gloss_processed{ corpusi } 		= cur_gloss;
	corpus_gloss_is_loan_sign( corpusi ) 	= 0;
end
loan_signs_gloss_list						= lower( textread( 'loan_signs.csv','%s', 'whitespace', '\n' ) );
for corpusi									= 1 : length( loan_signs_gloss_list );
	cur_gloss								= loan_signs_gloss_list{ corpusi };
	cur_gloss( strfind( cur_gloss,'-' ) ) 	= [ ];				cur_gloss( strfind( cur_gloss,'_' ) ) 	= [ ];
	loan_signs_gloss_list{ corpusi } 		= cur_gloss;
end
gloss_suffixes								= { '','fr','1fr','1' };
loan_signs_with_suffix						= { };
loanj										= 1;
for suffixi									= 1 : length( gloss_suffixes );
	for loani								= 1 : length( loan_signs_gloss_list );
		loan_signs_with_suffix{ loanj } 	= [ loan_signs_gloss_list{ loani },		gloss_suffixes{ suffixi } ];
		loanj 								= loanj + 1;
	end
end
for corpusi 									= 1 : length( corpus_gloss_processed );
	if 	~isempty( strmatch( corpus_gloss_processed{ corpusi }, 	loan_signs_with_suffix, 	'exact' ) );
		corpus_gloss_is_loan_sign( corpusi )	= 1;
		fprintf( 'loan_sign: %s\n', 			corpus_gloss_processed{ corpusi } );
	end
end
num_orig_gallaudet_corpus_gloss			= length( corpus_gloss );

extended_lexicon_corpus_gloss_start		= 4000;
extended_lexicon_corpus_gloss_id		= extended_lexicon_corpus_gloss_start;

brady_lexicon_corpus_gloss_start		= 5000;
brady_lexicon_corpus_gloss_id			= brady_lexicon_corpus_gloss_start;

lana_lexicon_corpus_gloss_start			= 10000;
lana_lexicon_corpus_gloss_id			= lana_lexicon_corpus_gloss_start;

dana_lexicon_corpus_gloss_start			= 14000;
dana_lexicon_corpus_gloss_id			= dana_lexicon_corpus_gloss_start;

new_signer_lexicon_corpus_gloss_start	= 16000;

dataset_text 							= { };
for dataseti 							= 1 : length( datasets );
	input_file 							= [ '../processed_annotations/',	datasets{ dataseti },	'.out' ];
	input_text 							= textread( input_file,				'%s', 	'whitespace', 	'\n' );
	dataset_text 						= { dataset_text{ : },				input_text{ : } };
end

compound_glosses 						= [ ];
signer_gloss_infos 						= [ ];
annotation_sections 					= strmatch( '9999999', 		dataset_text, 	'exact' );
annotation_sections( end + 1 ) 			= length( dataset_text ) 	+ 1;
for sectioni 							= 1 : length( annotation_sections )		- 1;
	section_annotation 					= { dataset_text{ annotation_sections( 	sectioni ) + 1 : annotation_sections( sectioni + 1 ) - 1 } };
	if	length( section_annotation )	< 4;	continue;	end
	
	clear( 'gloss_info' );
	
	%if	strfind( section_annotation{ 1 },	'../sept_07_2011/Brady_summer_2011/7-22-2011/SCENE5_THROW/THUMB/THUMBxml.3.xml' );	keyboard;	end
	
	gloss_info.utterance_xml				= [ xml_files_base_directory,	section_annotation{ 1 } ];
	xml_file_info							= dir( gloss_info.utterance_xml );
	gloss_info.xml_modified_datenum			= xml_file_info( 1 ).datenum;
	gloss_info.xml_modified_time			= uint32( sscanf( section_annotation{ 2 },	'%d' ) );
	
	utterance_video_parse					= regexpi( section_annotation{ 3 }, '([^\\/]+)[\\/]([^\\/]+)$', 'tokens' );
	if	length(  utterance_video_parse )	~= 	1	|| 	length( utterance_video_parse{ 1 } )	~= 2; 	dbstack; keyboard; continue;	end
	session									= utterance_video_parse{ 1 }{ 1 };
	utterance_video							= utterance_video_parse{ 1 }{ 2 };
	
	%[ session, utterance_video ]			= 	strtok( section_annotation{ 3 }, '[ /\ ]' );	utterance_video		= utterance_video( 2 : end );

	if 	length( session ) 	< 3				||	~strcmp( 	session( 1 : 3 ),	'ASL' );

		session_info_parse					= 	regexpi(	session, 	'DANA-VID-(20\d{2}_\d{2}_\d{2})$',  'tokens' );
		
		if	length(  session_info_parse )	~= 	1	|| 	length( session_info_parse{ 1 } )	~= 	1;
			session_info_parse					= 	regexpi( session,	'Dana_(20\d{2}_\d{2}_\d{2})$',  'tokens' );		end

		if	length(  session_info_parse )	== 	1	&& 	length( session_info_parse{ 1 } )	== 	1;
			session							= 	[ 'ASL_',	session_info_parse{ 1 }{ 1 } ];		end
	end
	
	if 	length( session ) 	< 3				||	~strcmp( 	session( 1 : 3 ),	'ASL' );
		
		utterance_xml_parse					= regexpi( 	section_annotation{ 1 }, '([^\\/]+)[\\/]([^\\/]+)$', 'tokens' );

		if	length(  utterance_xml_parse )	~= 	1	|| 	length( utterance_xml_parse{ 1 } )	~= 2;		continue;	end
		
		session_info_parse					= regexpi(	utterance_xml_parse{ 1 }{ 1 }, 	'(\d{4}_\d{2}_\d{2})-\d+$',  	'tokens' );

		if	length(  session_info_parse )	~= 	1	|| 	length( session_info_parse{ 1 } )	~= 1;
			fprintf( '%s	file://%s\n',	section_annotation{ 3 },	gloss_info.utterance_xml );		continue;	end
		
		session 							= [ 'ASL_',	session_info_parse{ 1 }{ 1 } ];
	end

	session 							= regexprep(  	session,	'-',				'_',	'ignorecase' );
	session 							= regexprep(  	session, 	'_[ld]ana$', 		'', 	'ignorecase' );
	session 							= regexprep(  	session, 	'_skin_segmented$',	'', 	'ignorecase' );
	session 							= regexprep(  	session, 	'_new_set$',		'', 	'ignorecase' );
	
	cur_gloss_is_new_liz_set			= strcmp( 		session, 'ASL_2010_08_10_Liz_additional_lexicon_signs' );
	cur_gloss_is_brady_set				= ~isempty( 	strfind( lower( session ), 	'brady' ) );
	cur_gloss_is_lana_set				= ~isempty( 	strmatch( upper( session ),  signer_sessions{ 5 } ) );
	cur_gloss_is_dana_set				= ~isempty( 	strmatch( upper( session ),  signer_sessions{ 6 } ) );

	cur_gloss_is_2011_annotations_set	= cur_gloss_is_brady_set | cur_gloss_is_lana_set | cur_gloss_is_dana_set;
	
	gloss_section1						= strmatch( '7777777', 	section_annotation, 	'exact' );
	gloss_section2						= strmatch( '5555555', 	section_annotation, 	'exact' );
	
	current_gloss						= '';
	if 		length( gloss_section1 ) 	> 0 && gloss_section1 < length( section_annotation ) && length( section_annotation{ gloss_section1 + 1 } ) > 0;
		current_gloss 					= section_annotation{ gloss_section1 + 1 };
		
	elseif 	length( gloss_section2 ) 	> 0 && gloss_section2 < length( section_annotation ) && length( section_annotation{ gloss_section2 + 1 } ) > 0;
		current_gloss 					= section_annotation{ gloss_section2 + 1 };
		
	elseif 	cur_gloss_is_2011_annotations_set;
		gloss_section1					= length( 	section_annotation ) + 1;
		gloss_section2					= gloss_section1;
	else
		fprintf(	'%s	 file://%s\n',	section_annotation{ 3 },	gloss_info.utterance_xml );		continue;
	end
	section_annotation 					= { section_annotation{ 1 : min( [ gloss_section1, 	gloss_section2 ] ) - 1 } };
	
	utterance_xml_filename 				= gloss_info.utterance_xml;
	if ~isempty( strfind( 				utterance_xml_filename,		'/' ) 	);
		utterance_xml_filename			= utterance_xml_filename( 	find( 	utterance_xml_filename 	== '/',	1,	'last' ) + 1 : 	end );
	end
	
	if isempty( find( 					utterance_xml_filename 		== 	'.' ) 	); 		continue;	end
	utterance_xml_filename				= utterance_xml_filename( 	1 :	find( 	utterance_xml_filename 	== '.',	1,	'first' ) - 1 );
	
	xml_str_index						= regexpi( 	utterance_xml_filename,		'xml$' 	);
	if	~isempty( xml_str_index ); 		utterance_xml_filename		= utterance_xml_filename( 1 : xml_str_index( 1 ) - 1 ); 		end
	
	if	cur_gloss_is_2011_annotations_set;
		xml_scene_str_index				= regexpi( 	utterance_xml_filename,		'^scene[0-9]*',	'end' );
		if	~isempty( xml_scene_str_index	);		utterance_xml_filename		= utterance_xml_filename( xml_scene_str_index( 1 ) + 1 : end ); end
	end
	
	is_utt_macro_unit_match				= min( [ 	strfind( utterance_xml_filename,	'Utt' 			),		...
		strfind( utterance_xml_filename,	'macro_unit_' 	),		...
		strfind( utterance_xml_filename,	'utter' 		)		]	);
	
	if isempty( is_utt_macro_unit_match )	||	is_utt_macro_unit_match 	~= 1; 	current_gloss	= utterance_xml_filename; 	end
	
	current_gloss							= lower( current_gloss );		current_gloss( strfind( current_gloss,' ' ) ) 	= '_';
	scene									= sscanf( utterance_video,		'scene%d-camera%d.mov' );
	if length( scene )	~=	2;				dbstack;		keyboard;		continue; 	end
	gloss_info.session						= session;
	gloss_info.scene						= scene( 1 );
	gloss_info.gloss						= current_gloss;
	gloss_info.gallaudet_gloss_description	= '';
	gloss_info.gloss_period					= [ ];
	gloss_info.onset_period					= [ ];
	gloss_info.offset_period				= [ ];
	gloss_info.initial_hold_period			= [ ];
	gloss_info.final_hold_period			= [ ];
	gloss_info.lhand_info					= [ ];
	gloss_info.rhand_info					= [ ];
	gloss_info.is_loan_sign					= 0;
	
	cur_corpus_gloss_id									= strmatch( gloss_info.gloss, 	corpus_gloss, 	'exact' );
	if 	isempty( cur_corpus_gloss_id )					|| 	cur_gloss_is_new_liz_set	|| cur_gloss_is_2011_annotations_set;
		cur_gloss										= gloss_info.gloss;
		cur_gloss( strfind( cur_gloss,'-' ) )			= [ ];	cur_gloss(	strfind( cur_gloss,'_' ) ) 	= [ ];
		cur_corpus_gloss_id								= strmatch( cur_gloss, corpus_gloss_processed, 	'exact' );
		if isempty( cur_corpus_gloss_id )				||	cur_gloss_is_new_liz_set	|| cur_gloss_is_2011_annotations_set;
			cur_corpus_gloss_id							= strmatch( cur_gloss, corpus_gloss_processed );
			if isempty( cur_corpus_gloss_id )			||	cur_gloss_is_new_liz_set	|| cur_gloss_is_2011_annotations_set;
				% fprintf( 'No matching gloss in corpus for %s, adding a new entry in the lexicon.\n', gloss_info.gloss );

				for corpus_glossi						= length( corpus_gloss ) + 1 	: dana_lexicon_corpus_gloss_id;
					corpus_gloss{ corpus_glossi }		= '';
				end

				if	cur_gloss_is_new_liz_set;
					gloss_info.gloss					= sprintf( 'EX:%s', gloss_info.gloss );
				elseif 	cur_gloss_is_brady_set;
					gloss_info.gloss					= sprintf( 'EX2:%s', gloss_info.gloss );
				elseif 	cur_gloss_is_lana_set;
					gloss_info.gloss					= sprintf( 'EX3:%s', gloss_info.gloss );
				elseif 	cur_gloss_is_dana_set;
					gloss_info.gloss					= sprintf( 'EX4:%s', gloss_info.gloss );
				end
				
				if	~cur_gloss_is_2011_annotations_set;
					
					cur_corpus_gloss_id									= extended_lexicon_corpus_gloss_id;
					corpus_gloss{ cur_corpus_gloss_id }					= gloss_info.gloss;
					corpus_gloss_is_loan_sign( cur_corpus_gloss_id )	= 0;
					extended_lexicon_corpus_gloss_id					= extended_lexicon_corpus_gloss_id + 1;
					if	extended_lexicon_corpus_gloss_id				>= brady_lexicon_corpus_gloss_start;
						fprintf( 'increase value of brady_lexicon_corpus_gloss_start \n' );		dbstack;	keyboard;
					end
					
				elseif	cur_gloss_is_brady_set;
					
					cur_corpus_gloss_id									= brady_lexicon_corpus_gloss_id;
					corpus_gloss{ cur_corpus_gloss_id }					= gloss_info.gloss;
					corpus_gloss_is_loan_sign( cur_corpus_gloss_id )	= 0;
					brady_lexicon_corpus_gloss_id						= brady_lexicon_corpus_gloss_id + 1;
					if	brady_lexicon_corpus_gloss_id					>= lana_lexicon_corpus_gloss_start;
						fprintf( 'increase value of lana_lexicon_corpus_gloss_start \n' );		dbstack;	keyboard;
					end
					
				elseif	cur_gloss_is_lana_set;
					
					cur_corpus_gloss_id									= lana_lexicon_corpus_gloss_id;
					corpus_gloss{ cur_corpus_gloss_id }					= gloss_info.gloss;
					corpus_gloss_is_loan_sign( cur_corpus_gloss_id )	= 0;
					lana_lexicon_corpus_gloss_id						= lana_lexicon_corpus_gloss_id + 1;
					if	lana_lexicon_corpus_gloss_id					>= dana_lexicon_corpus_gloss_start;
						fprintf( 'increase value of dana_lexicon_corpus_gloss_start \n' );		dbstack;	keyboard;
					end
					
				elseif	cur_gloss_is_dana_set;
					
					cur_corpus_gloss_id									= dana_lexicon_corpus_gloss_id;
					corpus_gloss{ cur_corpus_gloss_id }					= gloss_info.gloss;
					corpus_gloss_is_loan_sign( cur_corpus_gloss_id )	= 0;
					dana_lexicon_corpus_gloss_id						= dana_lexicon_corpus_gloss_id + 1;
					if	dana_lexicon_corpus_gloss_id					>= new_signer_lexicon_corpus_gloss_start;
						fprintf( 'increase value of new_signer_lexicon_corpus_gloss_start \n' );	dbstack;	keyboard;
					end
					
				else
					fprintf( 'This case has not been defined.' );		keyboard;
				end
				%continue;
			end
		end
	end
	cur_corpus_gloss_id					= cur_corpus_gloss_id( 1 );
	gloss_info.gloss					= corpus_gloss{ cur_corpus_gloss_id };
	gloss_info.is_loan_sign				= corpus_gloss_is_loan_sign( cur_corpus_gloss_id );
	
	signeri								= [ ];
	for signerj							= 1 : length( signer_sessions );
		if ~isempty( strmatch(			gloss_info.session, 	signer_sessions{ signerj } ) );
			signeri						= signerj; 		break;
		end
	end
	if isempty( signeri );
		fprintf( 'No matching signer session found for %s.\n', gloss_info.session ); continue;
	end
	
	subsection_ids						= { '1000', '2000', '10000' };
	subsections							= [ ];
	for i								= 1 : length( subsection_ids );
		t								= strmatch( subsection_ids{ i }, section_annotation, 'exact' );
		for tii							= 1 : length( t );
			tj							= t( tii );
			if tj						< length( section_annotation ) 	&& 	strcmp( section_annotation{ tj + 1 }, '8888888' );
				subsections( end + 1 ) 	= tj;
			end
		end
	end
	if 	isempty( subsections );			continue; 	end
	subsections( end + 1 )				= length( section_annotation ) + 1;
	subsections							= sort( subsections, 'ascend' );

	clear subsection_parsed_entities;
	time_scaling											= 2000;
	for subsectioni											= 1 : length( subsections )-1;
		subsection_id										= sscanf( section_annotation{ subsections( subsectioni ) }, '%d' );
		subsection_annotation								= { section_annotation{ subsections( subsectioni ) + 1 : subsections( subsectioni + 1 )-1 } };
		
		entities											= strmatch( '8888888', subsection_annotation, 'exact' );

		if isempty( entities );								continue; end

		entities( end + 1 )									= length( subsection_annotation ) + 1;
		clear parsed_entities;
		entities_start_times								= [ ];
		entityj												= 1;
		for entityi											= 1 : length( entities ) - 1;

			entity_subsection_annotation					= { subsection_annotation{ entities( entityi ) + 1 : entities( entityi + 1 )-1 } };
			items											= strmatch( '6666666', entity_subsection_annotation, 'exact' );
			items( end + 1 )								= length( entity_subsection_annotation ) + 1;
			entity_items									= { };
			itemk											= 0;
			for itemi										= 1 : length( items ) - 1;
				item_subsection								= { entity_subsection_annotation{ items( itemi ) + 1 : items( itemi + 1 )-1 } };
				if 	( subsection_id							== 10000 	&& 	length( item_subsection ) 	~= 4	) 	|| 	...
					( ( subsection_id						== 1000 	|| 	subsection_id == 2000 ) 	&& length( item_subsection ) ~= 3 );
					continue;
				end
				itemk										= itemk + 1;
				for itemj									= 1 : length( item_subsection );
					entity_items{ itemj,itemk }				= item_subsection{ itemj };
				end
			end
			if isempty( entity_items );						continue; end
			
			[ item_start_time, sort_order ]					= sort( str2double( { entity_items{ 2,: } } ) );
			entity_item_nrows								= size( entity_items,1 );
			parsed_entities( entityj ).entity_items 		= reshape( { entity_items{ :,sort_order } }, entity_item_nrows, [ ] );
			parsed_entities( entityj ).entity_start_time 	= item_start_time( 1 );
			parsed_entities( entityj ).entity_end_time 		= str2double( parsed_entities( entityj ).entity_items{ 1,end } );
			entities_start_times( entityj ) 				= item_start_time( 1 );
			parsed_entities( entityj ).num_items 			= size( entity_items,2 );
			entityj 										= entityj + 1;
		end
		[ entities_start_times, sort_order ]				= sort( entities_start_times );
		parsed_entities										= parsed_entities( sort_order );
		keep_entity											= [ ];
		unique_entity_start_times							= unique( entities_start_times );
		for unique_entityi									= 1 : length( unique_entity_start_times );
			match_entities									= find( entities_start_times == unique_entity_start_times( unique_entityi ) );
			[ t, max_match_entity ]							= max( [ parsed_entities( match_entities ).num_items ] );
			keep_entity( end + 1 )							= match_entities( max_match_entity );
		end
		parsed_entities										= parsed_entities( keep_entity );
		
		switch( subsection_id );
			case 10000
				subsectionj												= 1;
				
				onset_subentityi										= [ ];
				for onset_entityi										= 1 : length( parsed_entities );
					cur_entity_items									= { parsed_entities( onset_entityi ).entity_items{ end, : } };
					onset_subentityi									= strmatch( 'Onset', 	cur_entity_items,	'exact' );
					if length( onset_subentityi )						> 0; 	break; 		end
				end
				if	~isempty( onset_subentityi );
					subsection_num										= str2double( { parsed_entities( onset_entityi ).entity_items{ :, onset_subentityi } } );
					gloss_info.onset_period								= subsection_num( 2 : -1 : 1 ) / time_scaling;
					cur_entity_items									= { parsed_entities( onset_entityi ).entity_items{ :, onset_subentityi + 1 : end } };
					parsed_entities( onset_entityi ).entity_items 		= reshape( cur_entity_items,	4,	[ ] );
					parsed_entities( onset_entityi ).entity_start_time 	= sscanf( parsed_entities( onset_entityi ).entity_items{ 2, 1 }, '%d' );
				end

				initial_hold_subentityi									= [ ];
				if	gloss_info.xml_modified_datenum						> 734825;
					for initial_hold_entityi							= 1 : length( parsed_entities );
						cur_entity_items								= { parsed_entities( initial_hold_entityi ).entity_items{ end, : } };
						initial_hold_subentityi							= strmatch( 'Initial Hold', cur_entity_items,	'exact' );
						if length( initial_hold_subentityi )			> 0; 	break; 		end
					end
					if	~isempty( initial_hold_subentityi );
						subsection_num									= str2double( { parsed_entities( initial_hold_entityi ).entity_items{ :, initial_hold_subentityi } } );
						gloss_info.initial_hold_period					= subsection_num( 2 : -1 : 1 ) / time_scaling;
						cur_entity_items								= { parsed_entities( initial_hold_entityi ).entity_items{ :, initial_hold_subentityi + 1 : end } };
						parsed_entities( initial_hold_entityi ).entity_items 		= reshape( cur_entity_items,	4,	[ ] );
						parsed_entities( initial_hold_entityi ).entity_start_time 	= sscanf( parsed_entities( initial_hold_entityi ).entity_items{ 2, 1 }, '%d' );
					end
				end
				
				offset_subentityi										= [ ];
				for offset_entityi										= 1 : length( parsed_entities );
					cur_entity_items									= { parsed_entities( offset_entityi ).entity_items{ end, : } };
					offset_subentityi									= strmatch( 'Offset',	cur_entity_items,	'exact' );
					if length( offset_subentityi ) 						> 0; 	break; 		end
				end
				if	~isempty( offset_subentityi );
					subsection_num										= str2double( { parsed_entities( offset_entityi ).entity_items{ :,	offset_subentityi } } );
					gloss_info.offset_period							= subsection_num( 2 : -1 : 1 ) / time_scaling;
					cur_entity_items									= { parsed_entities( offset_entityi ).entity_items{ :, 1 : offset_subentityi - 1 } };
					parsed_entities( offset_entityi ).entity_items 		= reshape( cur_entity_items,	4,	[ ] );
					parsed_entities( offset_entityi ).entity_end_time 	= sscanf( parsed_entities( offset_entityi ).entity_items{ 1, end }, '%d' );
				end

				final_hold_subentityi									= [ ];
				if	gloss_info.xml_modified_datenum						> 734825;
					for final_hold_entityi								= 1 : length( parsed_entities );
						cur_entity_items								= { parsed_entities( final_hold_entityi ).entity_items{ end, : } };
						final_hold_subentityi							= strmatch( 'Final Hold',	cur_entity_items,	'exact' );
						if length( final_hold_subentityi ) 				> 0; 	break; 		end
					end
					if	~isempty( final_hold_subentityi );
						subsection_num									= str2double( { parsed_entities( final_hold_entityi ).entity_items{ :,	final_hold_subentityi } } );
						gloss_info.final_hold_period					= subsection_num( 2 : -1 : 1 ) / time_scaling;
						cur_entity_items								= { parsed_entities( final_hold_entityi ).entity_items{ :, 1 : final_hold_subentityi - 1 } };
						parsed_entities( final_hold_entityi ).entity_items 		= reshape( cur_entity_items,	4,	[ ] );
						parsed_entities( final_hold_entityi ).entity_end_time 	= sscanf( parsed_entities( final_hold_entityi ).entity_items{ 1, end }, '%d' );
					end
				end

				if		~isempty( initial_hold_subentityi )				&&	~isempty( final_hold_subentityi );
					parsed_entities 									= parsed_entities( initial_hold_entityi : final_hold_entityi );
				elseif	~isempty( initial_hold_subentityi )				&&	~isempty( offset_subentityi );
					parsed_entities 									= parsed_entities( initial_hold_entityi : offset_entityi );
				elseif	~isempty( initial_hold_subentityi );
					parsed_entities 									= parsed_entities( initial_hold_entityi : end );
				elseif	~isempty( onset_subentityi )					&&	~isempty( final_hold_subentityi );
					parsed_entities 									= parsed_entities( onset_entityi 		: final_hold_entityi );
				elseif	~isempty( final_hold_subentityi );
					parsed_entities 									= parsed_entities( 1 					: final_hold_entityi );
				elseif	~isempty( onset_subentityi )					&& 	~isempty( offset_subentityi );
					parsed_entities 									= parsed_entities( onset_entityi 		: offset_entityi );
				elseif	~isempty( onset_subentityi );
					parsed_entities 									= parsed_entities( onset_entityi 		: end 	);
				elseif	~isempty( offset_subentityi );
					parsed_entities 									= parsed_entities( 1 					: offset_entityi 	);
				end
				
			case 1000
				subsectionj						= 2;
			case 2000
				subsectionj						= 3;
			otherwise
				error( 'invalid subsection_id'	);
		end
		
		subsection_parsed_entities( subsectionj ).parsed_entities 	= parsed_entities;
	end
	
	if 	~exist( 'subsection_parsed_entities','var' )	|| 	length( subsection_parsed_entities( 1 ).parsed_entities ) 	== 0;
		continue;
	end
	
	for base_entityi							= 1 : length( subsection_parsed_entities( 1 ).parsed_entities );
		
		clear(	'entity_gloss_info'	);
		subsection_num							= str2double( { subsection_parsed_entities( 1 ).parsed_entities( base_entityi ).entity_items{ :,1 } } );
		entity_gloss_info.gloss_period			= subsection_num( 2 : -1 : 1 ) / time_scaling;
		entity_gloss_info.subgloss				= subsection_parsed_entities( 1 ).parsed_entities( base_entityi ).entity_items{ end,1 };
		gloss_entity_start_time					= subsection_parsed_entities( 1 ).parsed_entities( base_entityi ).entity_start_time;
		gloss_entity_end_time					= subsection_parsed_entities( 1 ).parsed_entities( base_entityi ).entity_end_time;
		entity_gloss_info.rhand_info			= [ ];
		entity_gloss_info.lhand_info			= [ ];
		
		for subsectionj							= 2 : length( subsection_parsed_entities );
			
			if isempty(	subsection_parsed_entities( subsectionj ).parsed_entities ); 	continue; 	end
			
			hand_entitiy_start_time				= [ subsection_parsed_entities( subsectionj ).parsed_entities( : ).entity_start_time 	];
			hand_entitiy_end_time				= [ subsection_parsed_entities( subsectionj ).parsed_entities( : ).entity_end_time 		];
			hand_entitiy_gloss_entity_overlap 	= [ ];
			for hand_entitiyi 					= 1 : length( subsection_parsed_entities( subsectionj ).parsed_entities );
				hand_entitiy_gloss_entity_overlap( hand_entitiyi )	=	min( hand_entitiy_end_time( hand_entitiyi ), 	gloss_entity_end_time 	) - ...
					max( hand_entitiy_start_time( hand_entitiyi ), 	gloss_entity_start_time );
			end
			[ best_overlap, hand_entityi ] 		= max( hand_entitiy_gloss_entity_overlap );
			%if length( subsection_parsed_entities( 1 ).parsed_entities ) > 1; fprintf( '' ); end
			if best_overlap 					< 0; 	continue; 	end
			
			handshape_entity_num 				= str2double( subsection_parsed_entities( subsectionj ).parsed_entities( hand_entityi ).entity_items );
			handshape_entity_num( 1 : 2,: ) 	= handshape_entity_num( 1 : 2,: ) / time_scaling;
			if subsectionj 						== 2;
				entity_gloss_info.rhand_info	= handshape_entity_num;
			else
				entity_gloss_info.lhand_info	= handshape_entity_num;
			end
		end
		
		if length( subsection_parsed_entities( 1 ).parsed_entities ) 		== 1;
			
			gloss_info.gloss_period						= entity_gloss_info.gloss_period;
			gloss_info.lhand_info						= entity_gloss_info.lhand_info;
			gloss_info.rhand_info						= entity_gloss_info.rhand_info;
			gloss_info.compound_gloss					= [ ];
			
			if	cur_gloss_is_2011_annotations_set		&&	~isempty( entity_gloss_info.subgloss );
				gloss_info.gallaudet_gloss_description	= entity_gloss_info.subgloss;
			end
			
			if ~isempty( gloss_info.rhand_info 	);
				gloss_info.gloss_period 				= [ gloss_info.rhand_info( 1,1 ), gloss_info.rhand_info( 2,end ) ];
			end
		else
			
			gloss_info.compound_gloss( base_entityi ).gloss_period			= entity_gloss_info.gloss_period;
			gloss_info.compound_gloss( base_entityi ).lhand_info			= entity_gloss_info.lhand_info;
			gloss_info.compound_gloss( base_entityi ).rhand_info			= entity_gloss_info.rhand_info;
			subgloss 														= entity_gloss_info.subgloss;
			
			%if isempty( strfind( subgloss, 'CL-' ) )						&&	isempty( strfind( subgloss, 'CL:' ) );
			%	subgloss 													= lower( subgloss );
			%end
			gloss_info.compound_gloss( base_entityi ).subgloss 				= subgloss;
			
			if ~isempty( gloss_info.compound_gloss( base_entityi ).rhand_info );
				gloss_info.compound_gloss( base_entityi ).gloss_period 		= 			...
					[ 	gloss_info.compound_gloss( base_entityi ).rhand_info( 1,1 ),	gloss_info.compound_gloss( base_entityi ).rhand_info( 2,end ) 	];
			end
		end
	end
	
	if length( gloss_info.compound_gloss ) 		> 0;
		subgloss_period 						= reshape( [ gloss_info.compound_gloss( : ).gloss_period ], 2, [ ] );
		gloss_info.gloss_period 				= [ min( subgloss_period( 1,: ) ), max( subgloss_period( 2,: ) ) ];
		compound_glosses( end + 1 ).gloss 		= gloss_info.gloss;
		compound_glosses( end ).subgloss 		= { gloss_info.compound_gloss.subgloss };
		%fprintf( '%15s %20s ',gloss_info.session, gloss_info.gloss ); fprintf( '%20s ',gloss_info.compound_gloss.subgloss ); fprintf( '\n' );
	end
	
	cur_corpus_gloss_id_saved_gloss 			= [ ];
	replace_gloss_info 							= 0;
	if signeri > length( signer_gloss_infos ) 	|| 	...
			cur_corpus_gloss_id 				> length( signer_gloss_infos( signeri ).gloss_infos ) 	||	...
			isempty( signer_gloss_infos( signeri ).gloss_infos( cur_corpus_gloss_id ).gloss );
		replace_gloss_info						= 1;
	else
		cur_corpus_gloss_id_saved_gloss 		= signer_gloss_infos( signeri 	).gloss_infos( cur_corpus_gloss_id );
	end
	
	if replace_gloss_info 						== 0;
		if isempty( cur_corpus_gloss_id_saved_gloss.compound_gloss )			&& ~isempty( gloss_info.compound_gloss );
			replace_gloss_info 					= 1;
		elseif ~isempty( cur_corpus_gloss_id_saved_gloss.compound_gloss )		&& isempty( gloss_info.compound_gloss );
			replace_gloss_info 					= 0;
		elseif cur_corpus_gloss_id_saved_gloss.xml_modified_time				< gloss_info.xml_modified_time;
			replace_gloss_info 					= 1;
		elseif strcmp( cur_corpus_gloss_id_saved_gloss.gloss, corpus_gloss{ cur_corpus_gloss_id } ) 	~= 1 	&& ...
				strcmp( gloss_info.gloss, corpus_gloss{ cur_corpus_gloss_id } ) 	== 1;
			replace_gloss_info 					= 1;
		elseif cur_corpus_gloss_id_saved_gloss.scene 							< gloss_info.scene 				||	...
				cur_corpus_gloss_id_saved_gloss.gloss_period( 1 ) 				< gloss_info.gloss_period( 1 );
			replace_gloss_info 					= 1;
		else
			%fprintf( 'Duplicate annotation.\n' ); %gloss_info %cur_corpus_gloss_id_saved_gloss
		end
	end
	
	if replace_gloss_info; 	signer_gloss_infos( signeri ).gloss_infos( cur_corpus_gloss_id ) 	= gloss_info; end
	
end

vid_handle													= [ ];
netavi_handle												= [ ];
prev_videofile												= '';

for singeri													= 1 : length( signer_gloss_infos );
	
	gloss_infos												= signer_gloss_infos( singeri ).gloss_infos;
	for glossi												= 1 : length( gloss_infos );
		if	~isempty(										gloss_infos( glossi ).gloss );
			cur_scene										= gloss_infos( glossi ).scene;
			cur_session										= gloss_infos( glossi ).session;
			break;
		end
	end
	cur_scene_start_glossi									= glossi;
	
	for glossi												= glossi 	: length( gloss_infos ) + 1;
		
		if 	glossi											<= length( gloss_infos ) &&	isempty( gloss_infos( glossi ).gloss );	continue; 	end
		
		if 	glossi											> length( gloss_infos ) ||	gloss_infos( glossi ).scene ~= cur_scene;
			
			scene_video_file								= sprintf( '%s/scene%d-camera1.vid', 	cur_session, 	cur_scene );
			cur_info.video_file								= get_full_ASL_vid_directory( scene_video_file ); 		openVideoFile;
			cur_scene_end_glossi							= glossi - 1;
			cur_scene_glossi								= [ cur_scene_start_glossi : cur_scene_end_glossi ];
			tt												= gloss_infos( cur_scene_glossi );
			
			for ttj											= length( tt ) : -1 : 1;
				if											~isempty( tt( ttj ).gloss ); 	break; 	end
			end
			
			if 	( 	tt( ttj ).gloss_period( 2 )				> 	vid_info.NumFrames	&&	...
					~isempty( strmatch( cur_session, 		{ 	'ASL_2008_06_10', 	'ASL_2008_08_04', 	'ASL_2008_08_13' } ) ) ) ||	...
				(	strcmp( cur_session, 'ASL_2008_08_13' ) && 	sum( cur_scene 	== 	[ 9, 12 ] 	) );

				fprintf( 'Scaling timecodes ( /= 2 ) for:	%s\n',	scene_video_file );	

				for tti										= 1 : length( tt );
					if isempty( tt( tti ).gloss ); 			continue; 	end
					tt( tti ).gloss_period					= floor( tt( tti ).gloss_period/2 	);
					tt( tti ).onset_period					= floor( tt( tti ).onset_period/2 	);
					tt( tti ).offset_period					= floor( tt( tti ).offset_period/2 	);
					if ~isempty( tt( tti ).rhand_info ); 	tt( tti ).rhand_info( 1 : 2,: )		= floor( tt( tti ).rhand_info( 1 : 2,: )/2 ); end
					if ~isempty( tt( tti ).lhand_info ); 	tt( tti ).lhand_info( 1 : 2,: )		= floor( tt( tti ).lhand_info( 1 : 2,: )/2 ); end
					if ~isempty( tt( tti ).compound_gloss 	);
						ttt									= tt( tti ).compound_gloss;
						for ttti							= 1 : length( ttt );
							ttt( ttti ).gloss_period 		= floor( ttt( ttti ).gloss_period/2 );
							if ~isempty( ttt( ttti ).rhand_info ); ttt( ttti ).rhand_info( 1 : 2,: ) = floor( ttt( ttti ).rhand_info( 1 : 2,: )/2 ); end
							if ~isempty( ttt( ttti ).lhand_info ); ttt( ttti ).lhand_info( 1 : 2,: ) = floor( ttt( ttti ).lhand_info( 1 : 2,: )/2 ); end
						end
						tt( tti ).compound_gloss 			= ttt;
					end
				end
				gloss_infos( cur_scene_glossi ) 			= tt;
			end
			
			if glossi										<= length( gloss_infos );
				cur_scene									= gloss_infos( glossi ).scene;
				cur_session									= gloss_infos( glossi ).session;
			end
			cur_scene_start_glossi							= glossi;
		end
		signer_gloss_infos( singeri ).gloss_infos			= gloss_infos;
	end
end

[ gloss_labels_from_movie, gallaudet_glosses ] 	= textread( gloss_translation_file, '%s %s', 'delimiter', ';', 'whitespace', '', 'endofline', '\r\n' );

gloss_labels_from_movie		= strrep( lower( gloss_labels_from_movie ), '-', '_' );

for singeri					= 1 : length( signer_gloss_infos );

	gloss_infos				= signer_gloss_infos( singeri ).gloss_infos;

	fp						= fopen( sprintf( '../processed_annotations/%s_handshapes.csv', signers{ singeri } ), 'w' );
	fprintf( fp,			'#,Sign label,Gallaudet gloss,Compound signs,Session,Scene,XML,MOV,Sign start,Sign end,' );
	fprintf( fp,			'Dominant handshape IDs,Non-dominant handshape IDs,Dominant handshape frames,Non-dominant handshape frames\n' );

	for glossi												= 1 : length( gloss_infos );

		gloss_info											= gloss_infos( glossi );
		if 	isempty( gloss_info.gloss )						continue; 	end
		
		if	~isempty( gloss_info.gallaudet_gloss_description );
			gallaudet_gloss_description						= gloss_info.gallaudet_gloss_description;
		else

			gallaudet_gloss_description						= '';
			cur_gloss										= strrep( lower( gloss_info.gloss ), '-', '_' );
			gallaudet_gloss_index							= strmatch( cur_gloss, 	gloss_labels_from_movie, 	'exact' );

			if isempty( gallaudet_gloss_index );
				gallaudet_gloss_index 						= strmatch( cur_gloss, 	gloss_labels_from_movie );
			end
			if isempty( gallaudet_gloss_index );
				gallaudet_gloss_index 						= strmatch( cur_gloss( 1 : end-1 ), 	gloss_labels_from_movie );
			end
			if ~isempty( gallaudet_gloss_index );

				gallaudet_gloss_index						= gallaudet_gloss_index( 1 );
				gallaudet_gloss_description					= gallaudet_glosses{ gallaudet_gloss_index };
				signer_gloss_infos( singeri ).gloss_infos( 	glossi ).gallaudet_gloss_description 	= gallaudet_gloss_description;
				gallaudet_gloss_description 				= strrep( gallaudet_gloss_description, 	',', 	'~' );
			else

				fprintf( 'Cannot find Gallaudet gloss descriptor for sign label: %s.\n',	cur_gloss	);
				EX_gloss_match_parse						= regexpi( 	cur_gloss,			'^EX[0-9]*:(.*)$',	'tokens' );
				if 	length(	EX_gloss_match_parse )			== 1		&& 	length( EX_gloss_match_parse{ 1 } )	==	1;
					gallaudet_gloss_description 			= upper(	EX_gloss_match_parse{ 1 }{ 1 } );
				else
					gallaudet_gloss_description 			= upper( 	cur_gloss 	);
				end
				signer_gloss_infos( singeri ).gloss_infos( 	glossi ).gallaudet_gloss_description 	= gallaudet_gloss_description;
			end
		end
		
		compound_gloss_str						= '';
		hand_strs								= { };
		hand_strs2								= { };
		for k									= 1 : 2;
			hand_strs{ k }						= ''; 			
			hand_strs2{ k } 					= '';
		end
		if isempty( gloss_info.compound_gloss );
			for k								= 1 : 2;
				if k							== 1;
					hand_info					= gloss_info.rhand_info;
				else
					hand_info					= gloss_info.lhand_info;
				end
				if isempty( hand_info );		continue; 		end
				for l							= 1 : size( hand_info,2 );
					white_spacing				= '^'; 	if l 	== size( hand_info,2 ); 	white_spacing 	= ''; 	end
					hand_strs{ k }				= sprintf( '%s%d%s', hand_strs{ k }, 	hand_info( 3,l ), 	white_spacing );
					hand_strs2{ k }				= sprintf( '%s%d%s', hand_strs2{ k }, 	hand_info( 1,l ), 	white_spacing );
				end
			end
		else

			for compoundi						= 1 : length( gloss_info.compound_gloss );
				linebreak						= ''; 	if compoundi 	~= 1; 	linebreak 	= '~'; 		end
				compound_gloss					= gloss_info.compound_gloss( compoundi );
				compound_gloss_str				= sprintf( '%s%s%s',	compound_gloss_str,	linebreak,compound_gloss.subgloss );
				for k							= 1 : 2;
					if k						== 1;
						hand_info				= compound_gloss.rhand_info;
					else
						hand_info				= compound_gloss.lhand_info;
					end
					if isempty( hand_info );	continue; end
					hand_strs{ k }				= sprintf( '%s%s',		hand_strs{ k }, 	linebreak );
					hand_strs2{ k }				= sprintf( '%s%s',		hand_strs2{ k }, 	linebreak );
					for l						= 1 : size( hand_info,2 );
						white_spacing			= '^'; 	if l 	== size( hand_info,2 ); 	white_spacing = ''; 	end
						hand_strs{ k }			= sprintf( '%s%d%s', 	hand_strs{ k }, 	hand_info( 3,l ), white_spacing );
						hand_strs2{ k }			= sprintf( '%s%d%s', 	hand_strs2{ k }, 	hand_info( 1,l ), white_spacing );
					end
				end
			end
		end

		session										= gloss_info.session( 5 : end );
		utterance_xml								= gloss_info.utterance_xml;
		utterance_xml								= strrep( utterance_xml, ' ', '^' );
		movie										= sprintf( 'mov/%s/scene%d-camera1.mov', gloss_info.session, gloss_info.scene );
		fprintf( fp,'%d,%s,%s,%s,%s,%d,%s,%s,', 	glossi, gloss_info.gloss, 	gallaudet_gloss_description, 	compound_gloss_str,	...
														session, gloss_info.scene, 	utterance_xml, movie );
		fprintf( fp, '%d,%d,%s,%s,%s,%s\n',  		gloss_info.gloss_period( 1 ), gloss_info.gloss_period( 2 ),						...
														hand_strs{ 1 }, hand_strs{ 2 }, hand_strs2{ 1 }, hand_strs2{ 2 } 	);
	end
	fclose( fp );
end

for singeri 								= 1 : length( signer_gloss_infos );
	fp2 									= fopen( sprintf( '../processed_annotations/%s_all_annotate.csv', signers{ singeri } ), 'w' );
	fprintf( fp2, 							'Dictionary index,Session and Scene,Sign gloss labels from the Gallaudet Dictionary,Start frame,End frame\n' );
	gloss_infos 							= signer_gloss_infos( singeri ).gloss_infos;
	cur_session 							= '';
	cur_scene 								= 0;
	fp1 									= [ ];	
	cur_session_id 							= 0;
	for glossi 								= 1 : length( gloss_infos );
		gloss_info 							= gloss_infos( glossi );
		if isempty( gloss_info.gloss );
			gloss_info.gloss										= '';
			gloss_info.session										= '';
			gloss_info.utterance_xml								= '';
			gloss_info.scene										= 0;
			gloss_info.gallaudet_gloss_description					= '';
			signer_gloss_infos( singeri ).gloss_infos( glossi ) 	= gloss_info;
			continue;
		end
		if ~strcmp( gloss_info.session, 	cur_session );
			if 	~isempty( fp1 ); 			fclose( fp1 ); 		end
			cur_session 					= gloss_info.session;
			fp1 							= fopen( sprintf( 	'../processed_annotations/%s.csv', cur_session ), 'w' );
			fprintf( fp2, '\n,%s,,,\n\n',	cur_session );
			cur_session_id 					= cur_session_id 	+ 1;
		end
		scene_str 							= '';
		if cur_scene 						~= gloss_info.scene;
			cur_scene						= gloss_info.scene;
			scene_str						= sprintf( 'scene%d-camera1.mov', cur_scene );
		end
		fprintf( fp1, '1,%d,%d,%d,%s\n', 	gloss_info.scene, gloss_info.gloss_period( 1 ), gloss_info.gloss_period( 2 ), gloss_info.gloss );
		fprintf( fp2, '%d,%s,%s,%d,%d\n', 	glossi, scene_str, gloss_info.gloss, gloss_info.gloss_period( 1 ), gloss_info.gloss_period( 2 ) );
	end
	fclose( fp1 );
	fclose( fp2 );
end

compound_glosses_gloss 						= { compound_glosses( : ).gloss };
[ compound_glosses_gloss, sort_order ] 		= sort( compound_glosses_gloss );
compound_glosses 							= compound_glosses( sort_order );
[ compound_glosses_gloss, uniq_index ] 		= unique( compound_glosses_gloss );
compound_glosses 							= compound_glosses( uniq_index );
corpus_gloss_compounds_unrolled				= { };
corpus_gloss_num_compound_parts				= [ ];
compounds_unrolledi 						= 1;
for corpusi 								= 1 : length( corpus_gloss );

	if isempty( strmatch( corpus_gloss{ corpusi }, 	compound_glosses_gloss, 	'exact' ) );
		corpus_gloss_compounds_unrolled{ compounds_unrolledi }		= corpus_gloss{ corpusi };
		corpus_gloss_num_compound_parts( corpusi )					= 0;
		compounds_unrolledi											= compounds_unrolledi + 1;
	else
		signer_compounds_length 									= [ ];
		for signeri 												= 1 : length( signer_gloss_infos );
			if length( signer_gloss_infos( signeri ).gloss_infos )	>= corpusi;
				signer_compounds_length( signeri )					= length( signer_gloss_infos( signeri ).gloss_infos( corpusi ).compound_gloss );
			end
		end
		if max( signer_compounds_length ) 	== 0; 					continue; 		end
		[ t,maxi ]													= max( signer_compounds_length );
		signer_compound												= signer_gloss_infos( maxi ).gloss_infos( corpusi ).compound_gloss;
		for compoundi												= 1 : length( signer_compound );
			corpus_gloss_compounds_unrolled{ compounds_unrolledi } 	= sprintf( '%s-@%d',corpus_gloss{ corpusi }, compoundi );
			compounds_unrolledi 									= compounds_unrolledi 	+ 1;
		end
		corpus_gloss_num_compound_parts( corpusi ) 					= length( signer_compound );
	end
end

agentic_compounds 	= [ ]; 					non_agentic_compounds 	= [ ];
for compoundi 								= 1 : length( compound_glosses );
	if ~isempty( strmatch( 'agent', 		compound_glosses( compoundi ).subgloss, 'exact' ) );
		agentic_compounds( end + 1 ) 		= compoundi;
	else
		non_agentic_compounds( end + 1 ) 	= compoundi;
	end
end

for compoundj 								= 1 : 2;
	if compoundj 							== 1;
		compounds 							= agentic_compounds;
	else
		compounds 							= non_agentic_compounds;
	end
	for compoundi 							= 1 : length( compounds );
		fprintf( '%s := ', 					compound_glosses( compounds( compoundi ) ).gloss );
		subgloss 							= compound_glosses( compounds( compoundi ) ).subgloss;
		fprintf( '%s + %s\n',				subgloss{ 1 },	subgloss{ 2 } );
	end
	fprintf( '\n' );
end

orig_signer_gloss_infos									= [ ];
extra_signer_gloss_infos								= [ ];
num_extra_signer_gloss_infos							= [ ];
brady_signer_gloss_infos								= [ ];
num_brady_signer_gloss_infos							= [ ];
lana_signer_gloss_infos									= [ ];
num_lana_signer_gloss_infos								= [ ];
dana_signer_gloss_infos									= [ ];
num_dana_signer_gloss_infos								= [ ];

for signeri												= 1 : length( signer_gloss_infos );

	orig_signer_gloss_infos( signeri ).gloss_infos		= signer_gloss_infos( signeri ).gloss_infos( 1 : min( num_orig_gallaudet_corpus_gloss, ...
															length( signer_gloss_infos( signeri ).gloss_infos ) ) );

	num_signeri_gloss_infos								= length( signer_gloss_infos( signeri ).gloss_infos );

	extended_lexicon_corpus_gloss_ids					= [ ];
	if	extended_lexicon_corpus_gloss_start				<= 	num_signeri_gloss_infos;
		extended_lexicon_corpus_gloss_ids				= [ extended_lexicon_corpus_gloss_start : 	... 
															min( num_signeri_gloss_infos, 		extended_lexicon_corpus_gloss_id ) ];
	end
	extra_signer_gloss_infos( signeri ).gloss_infos		= signer_gloss_infos( signeri ).gloss_infos( extended_lexicon_corpus_gloss_ids );
	num_extra_signer_gloss_infos( signeri )				= length( extended_lexicon_corpus_gloss_ids );

	brady_lexicon_corpus_gloss_ids						= [ ];
	if	brady_lexicon_corpus_gloss_start				<= 	num_signeri_gloss_infos;
		brady_lexicon_corpus_gloss_ids					= [ brady_lexicon_corpus_gloss_start  	: 	... 
															min( num_signeri_gloss_infos, 		brady_lexicon_corpus_gloss_id ) ];
	end
	brady_signer_gloss_infos( signeri ).gloss_infos		= signer_gloss_infos( signeri ).gloss_infos( brady_lexicon_corpus_gloss_ids );
	num_brady_signer_gloss_infos( signeri )				= length( brady_lexicon_corpus_gloss_ids );

	lana_lexicon_corpus_gloss_ids						= [ ];
	if	lana_lexicon_corpus_gloss_start					<= 	num_signeri_gloss_infos;
		lana_lexicon_corpus_gloss_ids					= [ lana_lexicon_corpus_gloss_start  	: 	... 
															min( num_signeri_gloss_infos, 		lana_lexicon_corpus_gloss_id ) ];
	end
	lana_signer_gloss_infos( signeri ).gloss_infos		= signer_gloss_infos( signeri ).gloss_infos( lana_lexicon_corpus_gloss_ids );
	num_lana_signer_gloss_infos( signeri )				= length( lana_lexicon_corpus_gloss_ids );

	dana_lexicon_corpus_gloss_ids						= [ ];
	if	dana_lexicon_corpus_gloss_start					<= 	num_signeri_gloss_infos;
		dana_lexicon_corpus_gloss_ids					= [ dana_lexicon_corpus_gloss_start  	: 	... 
															min( num_signeri_gloss_infos, 		dana_lexicon_corpus_gloss_id ) ];
	end
	dana_signer_gloss_infos( signeri ).gloss_infos		= signer_gloss_infos( signeri ).gloss_infos( dana_lexicon_corpus_gloss_ids );
	num_dana_signer_gloss_infos( signeri )				= length( dana_lexicon_corpus_gloss_ids );

end
signer_gloss_infos					= orig_signer_gloss_infos;

for signeri							= 1 : length( signer_gloss_infos );
	for glossi						= num_extra_signer_gloss_infos( signeri ) 	+ 1	: max( num_extra_signer_gloss_infos );
		extra_signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gloss		= '';
	end
	for glossi						= num_brady_signer_gloss_infos( signeri ) 	+ 1	: max( num_brady_signer_gloss_infos );
		brady_signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gloss		= '';
	end
	for glossi						= num_lana_signer_gloss_infos( signeri ) 	+ 1	: max( num_lana_signer_gloss_infos );
		lana_signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gloss		= '';
	end
	for glossi						= num_dana_signer_gloss_infos( signeri ) 	+ 1	: max( num_dana_signer_gloss_infos );
		dana_signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gloss		= '';
	end
end

for signeri							= 1 : length( brady_signer_gloss_infos );
	for glossi						= 1 : length( brady_signer_gloss_infos( signeri ).gloss_infos );
		if	~isempty( brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).utterance_xml );
			xml_file_info			= dir( brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).utterance_xml );
			cur_compound_gloss		= brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).compound_gloss;
			if	~isempty( cur_compound_gloss );
				cur_gallaudet_gloss_description		= cur_compound_gloss( 1 ).subgloss;
				for	compoundi		= 2 : length(  	cur_compound_gloss );
					cur_gallaudet_gloss_description	= [ cur_gallaudet_gloss_description, '+', cur_compound_gloss( compoundi ).subgloss ];
				end
				if	xml_file_info.datenum			> 734764;
					brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description	= cur_gallaudet_gloss_description;
				elseif	~strcmp( cur_gallaudet_gloss_description, brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description );
					fprintf( '%30s -> %s\n', brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).gloss, cur_gallaudet_gloss_description );
				end
			end
		end
		brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).corrected_sign_gloss	=	...
									brady_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description;
	end
end

for signeri							= 1 : length( lana_signer_gloss_infos );
	for glossi						= 1 : length( lana_signer_gloss_infos( signeri ).gloss_infos );
		if	~isempty( lana_signer_gloss_infos( signeri ).gloss_infos( glossi ).utterance_xml );
			cur_compound_gloss		= lana_signer_gloss_infos( signeri ).gloss_infos( glossi ).compound_gloss;
			if	~isempty( cur_compound_gloss );
				cur_gallaudet_gloss_description		= cur_compound_gloss( 1 ).subgloss;
				for	compoundi		= 2 : length(  	cur_compound_gloss );
					cur_gallaudet_gloss_description	= [ cur_gallaudet_gloss_description, '+', cur_compound_gloss( compoundi ).subgloss ];
				end
				lana_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description	= cur_gallaudet_gloss_description;
			end
		end
		lana_signer_gloss_infos( signeri ).gloss_infos( glossi ).corrected_sign_gloss	=	...
									lana_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description;
	end
end

for signeri							= 1 : length( dana_signer_gloss_infos );
	for glossi						= 1 : length( dana_signer_gloss_infos( signeri ).gloss_infos );
		if	~isempty( dana_signer_gloss_infos( signeri ).gloss_infos( glossi ).utterance_xml );
			cur_compound_gloss		= dana_signer_gloss_infos( signeri ).gloss_infos( glossi ).compound_gloss;
			if	~isempty( cur_compound_gloss );
				cur_gallaudet_gloss_description		= cur_compound_gloss( 1 ).subgloss;
				for	compoundi		= 2 : length(  	cur_compound_gloss );
					cur_gallaudet_gloss_description	= [ cur_gallaudet_gloss_description, '+', cur_compound_gloss( compoundi ).subgloss ];
				end
				dana_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description	= cur_gallaudet_gloss_description;
			end
		end
		dana_signer_gloss_infos( signeri ).gloss_infos( glossi ).corrected_sign_gloss	=	...
									dana_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description;
	end
end

%%
old_extra_signer_gloss_infos		= load( '../processed_annotations/may_03_2011.mat',	'extra_signer_gloss_infos' );
old_extra_signer_gloss_infos		= reorder_struct_fields( extra_signer_gloss_infos,	old_extra_signer_gloss_infos.extra_signer_gloss_infos );
old_extra_gloss_inf_glosses			= { };
old_extra_gloss_inf_xmls			= { };
for	old_extra_glossi				= 1 : length( old_extra_signer_gloss_infos( 1 ).gloss_infos );
	old_extra_gloss_inf_glosses{	old_extra_glossi, 1 }		= '';
	old_extra_gloss_inf_xmls{ 		old_extra_glossi, 1 }		= '';
	for	signeri						= 1 : length( old_extra_signer_gloss_infos );
		if	~isempty( old_extra_signer_gloss_infos( signeri ).gloss_infos( old_extra_glossi ).gloss );
			old_extra_gloss_inf_glosses{ old_extra_glossi, 1 }	= old_extra_signer_gloss_infos( signeri ).gloss_infos( old_extra_glossi ).gloss;
			old_extra_gloss_inf_xml		= old_extra_signer_gloss_infos( signeri ).gloss_infos( old_extra_glossi ).utterance_xml;
			old_extra_gloss_inf_xml		= regexp( old_extra_gloss_inf_xml, 	'([^\\/]+)[\\/]([^\\/]+)$', 'tokens' );
			old_extra_gloss_inf_xmls{ old_extra_glossi, 1 }		= [ old_extra_gloss_inf_xml{1}{1}, '/', old_extra_gloss_inf_xml{1}{2} ];
			break;
		end
	end
end

new_old_extra_gloss_map				= [ ];
for	new_extra_glossi				= 1 : length( extra_signer_gloss_infos( 1 ).gloss_infos );
	for	signeri						= 1 : length( old_extra_signer_gloss_infos );
		if	~isempty( extra_signer_gloss_infos( signeri ).gloss_infos( new_extra_glossi ).gloss );
			cur_new_extra_gloss		= extra_signer_gloss_infos( signeri ).gloss_infos( new_extra_glossi ).gloss;
			cur_new_extra_xml		= extra_signer_gloss_infos( signeri ).gloss_infos( new_extra_glossi ).utterance_xml;
			cur_new_extra_xml		= regexp( cur_new_extra_xml, '([^\\/]+)[\\/]([^\\/]+)$', 'tokens' );
			cur_new_extra_xml		= [ cur_new_extra_xml{ 1 }{ 1 },	'/', cur_new_extra_xml{ 1 }{ 2 } ];
			%cur_new_old_map		= strmatch( cur_new_extra_gloss,	old_extra_gloss_inf_glosses, 'exact' );
			cur_new_old_map			= strmatch( cur_new_extra_xml,		old_extra_gloss_inf_xmls, 	'exact' );
			if		length( cur_new_old_map )	== 1;
					new_old_extra_gloss_map( cur_new_old_map )	= new_extra_glossi;
			else	fprintf( '%d 	%d	%s\n', new_extra_glossi, length( cur_new_old_map ),	cur_new_extra_gloss ); 	end
			break;
		end
	end
end
% fprintf( '** %s\n',	old_extra_gloss_inf_glosses{ find( new_old_extra_gloss_map == 0 ) } );

clear(	'reordered_extra_gloss_infos',	'extra2_signer_gloss_infos' );
extra2_glosses						= setdiff( 1 : length( extra_signer_gloss_infos( 1 ).gloss_infos ), new_old_extra_gloss_map );
for	signeri							= 1 : length( extra_signer_gloss_infos );
	extra2_signer_gloss_infos( signeri ).gloss_infos	= extra_signer_gloss_infos( signeri ).gloss_infos( extra2_glosses );
end

for	signeri							= 1 : length( extra_signer_gloss_infos );
	for	new_old_extra_gloss_mapi	= 1 : length( new_old_extra_gloss_map );
		if	new_old_extra_gloss_map( new_old_extra_gloss_mapi )		~= 0;
			reordered_extra_gloss_infos( signeri ).gloss_infos(	new_old_extra_gloss_mapi )	=	...
									extra_signer_gloss_infos( signeri ).gloss_infos( new_old_extra_gloss_map( new_old_extra_gloss_mapi ) );

		elseif	signeri				<= length( old_extra_signer_gloss_infos );
			reordered_extra_gloss_infos( signeri ).gloss_infos(	new_old_extra_gloss_mapi )	=	...
										old_extra_signer_gloss_infos( signeri ).gloss_infos( new_old_extra_gloss_mapi );
		end
	end
end
extra_signer_gloss_infos			= reordered_extra_gloss_infos;
%%

%%
old_signer_gloss_infos				= load( '../processed_annotations/may_03_2011.mat',		'signer_gloss_infos' );
old_signer_gloss_infos				= reorder_struct_fields( signer_gloss_infos,			old_signer_gloss_infos.signer_gloss_infos );

for	glossi							= 1	: length( signer_gloss_infos( 1 ).gloss_infos );
	for	signeri						= 1 : length( old_signer_gloss_infos );

		if	isempty( 	signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gloss	) 	&& 			...
			isempty(	old_signer_gloss_infos( signeri ).gloss_infos( glossi ).gloss 	);	continue;	end

		if	~strcmp( 	signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gloss,					...
						old_signer_gloss_infos( signeri ).gloss_infos( glossi ).gloss 	) 	||			...
			~strcmp( 	signer_gloss_infos( 	signeri ).gloss_infos( glossi ).gallaudet_gloss_description,	...
						old_signer_gloss_infos( signeri ).gloss_infos( glossi ).gallaudet_gloss_description 	);

			if	isempty( signer_gloss_infos( signeri ).gloss_infos( glossi ).gloss );

				signer_gloss_infos( signeri ).gloss_infos( glossi )		= old_signer_gloss_infos( signeri ).gloss_infos( glossi );

			elseif isempty(	old_signer_gloss_infos( signeri ).gloss_infos( glossi ).gloss );

				extra2_signer_gloss_infos_glossi	= length( extra2_signer_gloss_infos( 1 ).gloss_infos )		+ 1;
				for	signerj							= 1 : length( extra2_signer_gloss_infos );
					extra2_signer_gloss_infos( signerj ).gloss_infos( extra2_signer_gloss_infos_glossi ).gloss	= '';
				end

				temp_gloss_info						= extra2_signer_gloss_infos( signeri ).gloss_infos( extra2_signer_gloss_infos_glossi );
				extra2_signer_gloss_infos( signeri ).gloss_infos( extra2_signer_gloss_infos_glossi )	= 	...
																		signer_gloss_infos( signeri ).gloss_infos( glossi );
				signer_gloss_infos( signeri ).gloss_infos( glossi )		= temp_gloss_info;
			else

				signer_gloss_infos( 	signeri ).gloss_infos( glossi )
				old_signer_gloss_infos( signeri ).gloss_infos( glossi )
				keyboard;
			end
		end
	end
end

%	gallaudet = {}; for i = 1:length( signer_gloss_infos ); gallaudet(:,i) = { signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
%	extra_gallaudet = {}; for i = 1:length( extra_signer_gloss_infos ); extra_gallaudet(:,i) = { extra_signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
%	old_extra_gallaudet = {}; for i = 1:length( old_extra_signer_gloss_infos ); old_extra_gallaudet(:,i) = { old_extra_signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
%	extra2_gallaudet = {}; for i = 1:length( extra2_signer_gloss_infos ); extra2_gallaudet(:,i) = { extra2_signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
% 	brady_gallaudet = {}; for i = 1:length( brady_signer_gloss_infos ); brady_gallaudet(:,i) = { brady_signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
% 	lana_gallaudet = {}; for i = 1:length( lana_signer_gloss_infos ); lana_gallaudet(:,i) = { lana_signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
% 	dana_gallaudet = {}; for i = 1:length( dana_signer_gloss_infos ); dana_gallaudet(:,i) = { dana_signer_gloss_infos( i ).gloss_infos( : ).gallaudet_gloss_description }'; end
%%

if 	1;
	matlab_save_file	= sprintf( 		'../processed_annotations/%s.mat',	datasets{ 1 } );			% all_sessions.mat
	save(		matlab_save_file, 		'signer_gloss_infos',		'extra_signer_gloss_infos',	  		'extra2_signer_gloss_infos',		...
				'corpus_gloss',			'compound_glosses',			'corpus_gloss_num_compound_parts', 	'corpus_gloss_compounds_unrolled', 	...
				'agentic_compounds', 	'non_agentic_compounds' 	);

	new_signers_save_file	= sprintf( 	'../processed_annotations/%s_new_signers_%.5f.mat',	datasets{ 1 },  now( ) );
	save(	new_signers_save_file,		'brady_signer_gloss_infos', 	'lana_signer_gloss_infos',		'dana_signer_gloss_infos' );
end
