Update with latest DDK and support OS X 10.11 format

DictUnifierAppDelegate.m

@@ -342,7 +342,7 @@ - (void) startBuildingWith: (NSString *) dictName
     NSInteger versionMinor = [[[productVersion componentsSeparatedByString:@"."] lastObject] integerValue];
     if (versionMinor >= 6) {
         [arguments insertObject: @"-v"   atIndex: 0];
-        [arguments insertObject: @"10.6" atIndex: 1];
+        [arguments insertObject: versionMinor >= 11 ? @"10.11" : @"10.6" atIndex: 1];
         NSLog(@"%@", arguments);
     }
 

ddk/build_dict.sh

@@ -4,11 +4,11 @@
 #
 #
 
-DICT_BUILD_TOOL_DIR=${DICT_BUILD_TOOL_DIR:-"/Developer/Extras/Dictionary Development Kit"}
-DICT_BUILD_TOOL_BIN="$DICT_BUILD_TOOL_DIR/bin"
+DICT_BUILD_TOOL_BIN=$(cd "$(dirname "$0")"; pwd)		# The directory that contains this script.
 
-do_add_supplementary_key=1
-preserve_unused_ref_id_in_reference_index=0
+
+do_add_supplementary_key=${do_add_supplementary_key:-1}
+preserve_unused_ref_id_in_reference_index=${preserve_unused_ref_id_in_reference_index:-0}
 
 COMPRESS_OPT=
 COMPRESS_SETTING=0
@@ -52,7 +52,7 @@ CSS_NAME="$3"
 PLIST_NAME="$4"
 
 #Fix outdated options of our dictionaries (should be removed after dictionaries updated)
-if [ $COMPRESS_SETTING -eq 2 ] && [ $TRIE_SETTING -eq 1 ] && [ $COMPATIBLE_VERS = "10.5" ]; then
+if [ $COMPRESS_SETTING -eq 2 ] && [ $TRIE_SETTING -gt 0 ] && [ $COMPATIBLE_VERS = "10.5" ]; then
 	COMPATIBLE_VERS="10.6"
 fi
 
@@ -62,11 +62,13 @@ if [ $COMPATIBLE_VERS = "10.5" ]; then
 		COMPRESS_OPT="-c 1"
 	fi
 	TRIE_OPT=""
-elif [ $COMPATIBLE_VERS = "10.6" ]; then
+elif [ $COMPATIBLE_VERS = "10.6" ] || [ $COMPATIBLE_VERS = "10.11" ]; then
 	if [ $TRIE_SETTING -eq 0 ]; then
 		TRIE_OPT="-t 1"
 	elif [ $TRIE_SETTING -eq 1 ]; then
 		TRIE_OPT="-t 2"
+	elif [ $TRIE_SETTING -eq 2 ]; then
+		TRIE_OPT="-t 3"
 	fi
 	if [ ! -n "$COMPRESS_OPT" ]; then
 		COMPRESS_OPT="-c 2"
@@ -76,6 +78,11 @@ else
 	exit 1
 fi
 
+CONTENTS_DATA_PATH=Contents
+if [ $COMPATIBLE_VERS = "10.11" ]; then
+	CONTENTS_DATA_PATH="Contents/Resources"
+fi
+
 DICT_DEV_KIT_OBJ_DIR=${DICT_DEV_KIT_OBJ_DIR:-objects}
 export DICT_DEV_KIT_OBJ_DIR
 
@@ -109,104 +116,108 @@ plutil -s "$PLIST_NAME" || error "Error."
 tr "\r" "\n" < "$PLIST_NAME" > $OBJECTS_DIR/dict.plist || error "Error."
 
 # Merge property to dictionary template.
-xsltproc "$DICT_BUILD_TOOL_BIN"/extract_property.xsl $OBJECTS_DIR/dict.plist > $OBJECTS_DIR/dict_prop_list.txt || error "Error."
-rm $OBJECTS_DIR/dict.plist
-"$DICT_BUILD_TOOL_BIN"/generate_dict_template.sh $COMPRESS_OPT $ENCRYPT_OPT $TRIE_OPT $OBJECTS_DIR/dict_prop_list.txt > $OBJECTS_DIR/customized_template.plist || error "Error."
-plutil -s $OBJECTS_DIR/customized_template.plist || error "Error."
-rm $OBJECTS_DIR/dict_prop_list.txt
+xsltproc "$DICT_BUILD_TOOL_BIN"/extract_property.xsl "$OBJECTS_DIR"/dict.plist > "$OBJECTS_DIR"/dict_prop_list.txt || error "Error."
+rm "$OBJECTS_DIR"/dict.plist
+IDX_DICT_VERS=""
+if [ $COMPATIBLE_VERS = "10.11" ]; then
+	IDX_DICT_VERS="-v 3"
+fi
+"$DICT_BUILD_TOOL_BIN"/generate_dict_template.sh $COMPRESS_OPT $ENCRYPT_OPT $TRIE_OPT $IDX_DICT_VERS "$OBJECTS_DIR"/dict_prop_list.txt > "$OBJECTS_DIR"/customized_template.plist || error "Error."
+plutil -s "$OBJECTS_DIR"/customized_template.plist || error "Error."
+rm "$OBJECTS_DIR"/dict_prop_list.txt
 
 # Preprocess sources.
 echo "- Preprocessing dictionary sources."
-tr "\r" "\n" < "$SRC_FILE" > $OBJECTS_DIR/dict.xml || error "Error."
+tr "\r" "\n" < "$SRC_FILE" > "$OBJECTS_DIR"/dict.xml || error "Error."
 # Replace localizable <index> format to the standard one
 sed 's/<d:index>[[:blank:]\n]*<d:index_value>\([^<]*\)<\/d:index_value>[[:blank:]\n]*<d:index_title>\([^<]*\)<\/d:index_title>[[:blank:]\n]*<\/d:index>/<d:index d:value="\1" d:title="\2"\/>/g' $OBJECTS_DIR/dict.xml > $OBJECTS_DIR/dict_mod.xml || error "Error."
-rm $OBJECTS_DIR/dict.xml
-"$DICT_BUILD_TOOL_BIN"/make_line.pl $OBJECTS_DIR/dict_mod.xml > $OBJECTS_DIR/dict.formattedSource.xml || error "Error."
-rm $OBJECTS_DIR/dict_mod.xml
-"$DICT_BUILD_TOOL_BIN"/make_body.pl $OBJECTS_DIR/dict.formattedSource.xml || error "Error."
+rm "$OBJECTS_DIR"/dict.xml
+"$DICT_BUILD_TOOL_BIN"/make_line.pl "$OBJECTS_DIR"/dict_mod.xml > "$OBJECTS_DIR"/dict.formattedSource.xml || error "Error."
+rm "$OBJECTS_DIR"/dict_mod.xml
+"$DICT_BUILD_TOOL_BIN"/make_body.pl "$OBJECTS_DIR"/dict.formattedSource.xml || error "Error."
 # The make_body.pl creates $OBJECTS_DIR/dict.body and $OBJECTS_DIR/dict.offset
 
 
 # Extract index data.
 echo "- Extracting index data."
-"$DICT_BUILD_TOOL_BIN"/extract_index.pl $OBJECTS_DIR/dict.formattedSource.xml > $OBJECTS_DIR/key_entry_list.txt || error "Error."
-"$DICT_BUILD_TOOL_BIN"/extract_referred_id.pl $OBJECTS_DIR/dict.formattedSource.xml > $OBJECTS_DIR/referred_id_list.txt || error "Error."
-"$DICT_BUILD_TOOL_BIN"/extract_front_matter_id.pl "$PLIST_NAME" >> $OBJECTS_DIR/referred_id_list.txt || error "Error."
-rm $OBJECTS_DIR/dict.formattedSource.xml
+"$DICT_BUILD_TOOL_BIN"/extract_index.pl "$OBJECTS_DIR"/dict.formattedSource.xml > "$OBJECTS_DIR"/key_entry_list.txt || error "Error."
+"$DICT_BUILD_TOOL_BIN"/extract_referred_id.pl "$OBJECTS_DIR"/dict.formattedSource.xml > "$OBJECTS_DIR"/referred_id_list.txt || error "Error."
+"$DICT_BUILD_TOOL_BIN"/extract_front_matter_id.pl "$PLIST_NAME" >> "$OBJECTS_DIR"/referred_id_list.txt || error "Error."
+rm "$OBJECTS_DIR"/dict.formattedSource.xml
 
 
 ########
 
 # Prepare dictionary bundle.
 echo "- Preparing dictionary bundle."
-"$DICT_BUILD_TOOL_BIN"/make_dict_package $OBJECTS_DIR/dict.dictionary $OBJECTS_DIR/customized_template.plist || error "Error."
-rm $OBJECTS_DIR/customized_template.plist
+"$DICT_BUILD_TOOL_BIN"/make_dict_package "$OBJECTS_DIR"/dict.dictionary "$OBJECTS_DIR"/customized_template.plist || error "Error."
+rm "$OBJECTS_DIR"/customized_template.plist
 
 # Add body reocrd to dictionary.
 echo "- Adding body data."
-"$DICT_BUILD_TOOL_BIN"/add_body_record $OBJECTS_DIR/dict.dictionary $BODY_DATA_NAME $OBJECTS_DIR/dict.offsets $OBJECTS_DIR/dict.body > $OBJECTS_DIR/entry_body_list.txt || error "Error."
-# rm $OBJECTS_DIR/dict.offsets
-# rm $OBJECTS_DIR/dict.body
+"$DICT_BUILD_TOOL_BIN"/add_body_record "$OBJECTS_DIR"/dict.dictionary $BODY_DATA_NAME "$OBJECTS_DIR"/dict.offsets "$OBJECTS_DIR"/dict.body > "$OBJECTS_DIR"/entry_body_list.txt || error "Error."
+# rm "$OBJECTS_DIR"/dict.offsets
+# rm "$OBJECTS_DIR"/dict.body
 
 
 # Make key body matching list
 echo "- Preparing index data."
-"$DICT_BUILD_TOOL_BIN"/replace_entryid_bodyid.pl $OBJECTS_DIR/entry_body_list.txt < $OBJECTS_DIR/key_entry_list.txt > $OBJECTS_DIR/key_body_list.txt || error "Error."
-rm $OBJECTS_DIR/key_entry_list.txt
+"$DICT_BUILD_TOOL_BIN"/replace_entryid_bodyid.pl "$OBJECTS_DIR"/entry_body_list.txt < "$OBJECTS_DIR"/key_entry_list.txt > "$OBJECTS_DIR"/key_body_list.txt || error "Error."
+rm "$OBJECTS_DIR"/key_entry_list.txt
 
 # Normalize key_text
 # "$DICT_BUILD_TOOL_BIN"/normalize_key_text.pl < $OBJECTS_DIR/key_body_list.txt > $OBJECTS_DIR/normalized_key_body_list.txt || error "Error."
-"$DICT_BUILD_TOOL_BIN"/normalize_key_text < $OBJECTS_DIR/key_body_list.txt > $OBJECTS_DIR/normalized_key_body_list_1.txt || error "Error."
-rm $OBJECTS_DIR/key_body_list.txt
+"$DICT_BUILD_TOOL_BIN"/normalize_key_text < "$OBJECTS_DIR"/key_body_list.txt > "$OBJECTS_DIR"/normalized_key_body_list_1.txt || error "Error."
+rm "$OBJECTS_DIR"/key_body_list.txt
 
 if [ $do_add_supplementary_key -gt 0 ]
 then
-	"$DICT_BUILD_TOOL_BIN"/add_supplementary_key < $OBJECTS_DIR/normalized_key_body_list_1.txt > $OBJECTS_DIR/normalized_key_body_list_2.txt
-	rm $OBJECTS_DIR/normalized_key_body_list_1.txt || error "Error."
+	"$DICT_BUILD_TOOL_BIN"/add_supplementary_key < "$OBJECTS_DIR"/normalized_key_body_list_1.txt > "$OBJECTS_DIR"/normalized_key_body_list_2.txt
+	rm "$OBJECTS_DIR"/normalized_key_body_list_1.txt || error "Error."
 else
-	mv $OBJECTS_DIR/normalized_key_body_list_1.txt $OBJECTS_DIR/normalized_key_body_list_2.txt || error "Error."
+	mv "$OBJECTS_DIR"/normalized_key_body_list_1.txt "$OBJECTS_DIR"/normalized_key_body_list_2.txt || error "Error."
 fi
 
-"$DICT_BUILD_TOOL_BIN"/remove_duplicate_key.pl < $OBJECTS_DIR/normalized_key_body_list_2.txt > $OBJECTS_DIR/normalized_key_body_list.txt
-rm $OBJECTS_DIR/normalized_key_body_list_2.txt || error "Error."
+"$DICT_BUILD_TOOL_BIN"/remove_duplicate_key.pl < "$OBJECTS_DIR"/normalized_key_body_list_2.txt > "$OBJECTS_DIR"/normalized_key_body_list.txt
+rm "$OBJECTS_DIR"/normalized_key_body_list_2.txt || error "Error."
 
 
 # Add key_text index record to dictionary.
 echo "- Building key_text index."
-"$DICT_BUILD_TOOL_BIN"/build_key_index $OBJECTS_DIR/dict.dictionary $KEY_TEXT_INDEX_NAME $OBJECTS_DIR/normalized_key_body_list.txt $COMPATIBLE_VERS || error "Error."
+"$DICT_BUILD_TOOL_BIN"/build_key_index "$OBJECTS_DIR"/dict.dictionary $KEY_TEXT_INDEX_NAME "$OBJECTS_DIR"/normalized_key_body_list.txt $COMPATIBLE_VERS || error "Error."
 # "$DICT_BUILD_TOOL_BIN"/add_key_index_record $OBJECTS_DIR/dict.dictionary $KEY_TEXT_INDEX_NAME $OBJECTS_DIR/normalized_key_body_list.txt || error "Error."
 # rm $OBJECTS_DIR/normalized_key_body_list.txt
 
 # Add entry_id index record to dictionary.
 echo "- Building reference index."
 if [ $preserve_unused_ref_id_in_reference_index -gt 0 ]
 then
-	"$DICT_BUILD_TOOL_BIN"/build_reference_index $OBJECTS_DIR/dict.dictionary $ENTRY_ID_INDEX_NAME $OBJECTS_DIR/entry_body_list.txt || error "Error."
+	"$DICT_BUILD_TOOL_BIN"/build_reference_index "$OBJECTS_DIR"/dict.dictionary $ENTRY_ID_INDEX_NAME "$OBJECTS_DIR"/entry_body_list.txt || error "Error."
 else
-	"$DICT_BUILD_TOOL_BIN"/pick_referred_entry_id.pl $OBJECTS_DIR/referred_id_list.txt < $OBJECTS_DIR/entry_body_list.txt > $OBJECTS_DIR/referred_entry_body_list.txt || error "Error."
-	"$DICT_BUILD_TOOL_BIN"/build_reference_index $OBJECTS_DIR/dict.dictionary $ENTRY_ID_INDEX_NAME $OBJECTS_DIR/referred_entry_body_list.txt || error "Error."
+	"$DICT_BUILD_TOOL_BIN"/pick_referred_entry_id.pl "$OBJECTS_DIR"/referred_id_list.txt < "$OBJECTS_DIR"/entry_body_list.txt > "$OBJECTS_DIR"/referred_entry_body_list.txt || error "Error."
+	"$DICT_BUILD_TOOL_BIN"/build_reference_index "$OBJECTS_DIR"/dict.dictionary $ENTRY_ID_INDEX_NAME "$OBJECTS_DIR"/referred_entry_body_list.txt || error "Error."
 fi
 # "$DICT_BUILD_TOOL_BIN"/add_reference_index_record $OBJECTS_DIR/dict.dictionary $ENTRY_ID_INDEX_NAME $OBJECTS_DIR/entry_body_list.txt || error "Error."
 # rm $OBJECTS_DIR/entry_body_list.txt
 
 # Make the dictioanry read-only
 echo "- Fixing dictionary property."
-mv $OBJECTS_DIR/dict.dictionary/Contents/Info.plist $OBJECTS_DIR/Info.plist
-plutil -convert xml1 $OBJECTS_DIR/Info.plist
-"$DICT_BUILD_TOOL_BIN"/make_readonly.pl < $OBJECTS_DIR/Info.plist > $OBJECTS_DIR/dict.dictionary/Contents/Info.plist || error "Error."
-rm $OBJECTS_DIR/Info.plist
-plutil -convert binary1 $OBJECTS_DIR/dict.dictionary/Contents/Info.plist
+mv "$OBJECTS_DIR"/dict.dictionary/Contents/Info.plist "$OBJECTS_DIR"/Info.plist
+plutil -convert xml1 "$OBJECTS_DIR"/Info.plist
+"$DICT_BUILD_TOOL_BIN"/make_readonly.pl < "$OBJECTS_DIR"/Info.plist > "$OBJECTS_DIR"/dict.dictionary/Contents/Info.plist || error "Error."
+rm "$OBJECTS_DIR"/Info.plist
+plutil -convert binary1 "$OBJECTS_DIR"/dict.dictionary/Contents/Info.plist
 
 # Copy other files.
 echo "- Copying CSS."
-cp -f "$CSS_NAME" $OBJECTS_DIR/dict.dictionary/Contents/DefaultStyle.css || error "Error."
+cp -f "$CSS_NAME" "$OBJECTS_DIR"/dict.dictionary/"$CONTENTS_DATA_PATH"/DefaultStyle.css || error "Error."
 if [ -d "$OTHER_RSRC_DIR" ] && [ -n "`ls $OTHER_RSRC_DIR`" ]
 then
 	echo "- Copying other resources."
-	cp -XRf "$OTHER_RSRC_DIR"/* $OBJECTS_DIR/dict.dictionary/Contents || error "Error."
+	cp -XRf "$OTHER_RSRC_DIR"/* "$OBJECTS_DIR"/dict.dictionary/"$CONTENTS_DATA_PATH" || error "Error."
 fi
 
-mv -f $OBJECTS_DIR/dict.dictionary "$OBJECTS_DIR/$DICT_NAME.dictionary" || error "Error."
+mv -f "$OBJECTS_DIR"/dict.dictionary "$OBJECTS_DIR/$DICT_NAME.dictionary" || error "Error."
 
 #
 echo "- Finished building $OBJECTS_DIR/$DICT_NAME.dictionary."

ddk/extract_referred_id.pl

@@ -58,15 +58,19 @@ sub process_an_entry
 	my ( $entry ) = @_;
 	# printf "== [%s]\n", $entry;
 
-
 	my $text = $entry;
-	while ( $text =~ /(<$a_ndnm$reqsp$href$optsp=$optsp($qattr_ne)$optsp>.*?<\/$a_ndnm>)/ )
+
+	while ( $text =~ /((<$a_ndnm$reqsp[^>]+>).*?<\/$a_ndnm>)/ )
 	{
-		$text		= $';	#'
-		my $reference		= $1;
-		$2 =~ /^(["']{1})([^\1]*)\1$/;
-		my $referred_url	= $2;
-		process_a_referred_url( $referred_url );
+		$text			= $';	#'
+		#my $reference	= $1;
+		my $a_open_tag	= $2;
+		if ( $a_open_tag =~ /$reqsp$href$optsp=$optsp($qattr_ne)/ )
+		{
+			$1 =~ /^(["']{1})([^\1]*)\1$/;
+			my $referred_url	= $2;
+			process_a_referred_url( $referred_url );
+		}
 	}
 }
 

ddk/generate_dict_template.sh

@@ -10,14 +10,16 @@ dictionary_vers=1
 compress_body=1
 body_id_size=4
 encrypt_body=0
-compress_triedata=0
-burst_trie=0
+compress_heapdata=0
+compress_indexdata=0
+compress_trie=0
 
 COMPRESS_OPT=
 ENCRYPT_OPT=
 TRIE_OPT=
+IDX_DICT_VERS=1
 
-while getopts c:e:t: opt
+while getopts c:e:t:v: opt
 do
 	case $opt in
 	c)
@@ -43,14 +45,24 @@ do
 		TRIE_OPT=$OPTARG
 		if [ $TRIE_OPT -gt 0 ]; then
 			dictionary_vers=2
-			burst_trie=1
-			if [ $TRIE_OPT -eq 2 ]; then
-				compress_triedata=1
+			compress_trie=1
+			if [ $TRIE_OPT -eq 3 ]; then
+			    compress_indexdata=1
+				compress_trie=2
+			fi
+			if [ $TRIE_OPT -gt 1 ]; then
+				compress_heapdata=1
 			fi
 		fi
 		;;
+	v)
+		IDX_DICT_VERS=$OPTARG
+		;;
 	esac
 done
+if [ $IDX_DICT_VERS -gt 2 ]; then
+	dictionary_vers=$IDX_DICT_VERS
+fi
 
 shift $((OPTIND - 1))
 
@@ -92,6 +104,10 @@ cat << END_OF_FILE
 						<string>DCSBodyDataIndex</string>
 					</dict>
 				</array>
+END_OF_FILE
+if [ $compress_indexdata -eq 0 ]
+then
+cat << END_OF_FILE 				
 				<key>IDXFixedDataFields</key>
 				<array>
 					<dict>
@@ -134,6 +150,9 @@ cat << END_OF_FILE
 						<integer>2</integer>
 					</dict>
 				</array>
+END_OF_FILE
+fi
+cat << END_OF_FILE 				
 			</dict>
 			<key>IDXIndexDataSizeLength</key>
 			<integer>2</integer>
@@ -159,7 +178,7 @@ cat << END_OF_FILE
 				<string>KeyText.data</string>
 END_OF_FILE
 
-if [ $compress_triedata -gt 0 ]
+if [ $compress_heapdata -gt 0 ]
 then
 cat << END_OF_FILE
 				<key>HeapDataCompressionType</key>
@@ -171,11 +190,11 @@ cat << END_OF_FILE
 			</dict>
 END_OF_FILE
 
-if [ $burst_trie -gt 0 ]
+if [ $compress_trie -gt 0 ]
 then
 cat << END_OF_FILE
 			<key>TrieIndexCompressionType</key>
-			<integer>1</integer>
+			<integer>$compress_trie</integer>
 END_OF_FILE
 fi
 
@@ -221,11 +240,11 @@ cat << END_OF_FILE
 			</dict>
 END_OF_FILE
 
-if [ $burst_trie -gt 0 ]
+if [ $compress_trie -gt 0 ]
 then
 cat << END_OF_FILE
 			<key>TrieIndexCompressionType</key>
-			<integer>1</integer>
+			<integer>$compress_trie</integer>
 END_OF_FILE
 fi
 

ddk/normalize_key_text.pl

@@ -24,7 +24,7 @@
 		next;
 	}
 
-	my ( $key_text, $body_id, $flags, $title, $anchor, $yomi, %entry_title ) = split /\t/, $record;
+	my ( $key_text, $body_id, $flags, $title, $anchor, $yomi, $entry_title ) = split /\t/, $record;
 	if ( not defined $anchor )
 	{
 		printf STDERR "*** Unknown format. Skipped [%s]\n", $record;
@@ -33,7 +33,7 @@
 
 	my $normalized_key = createSearchKey( $key_text );
 	printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", 
-		$normalized_key, $body_id, $flags, $title, $anchor, $yomi, %entry_title;
+		$normalized_key, $body_id, $flags, $title, $anchor, $yomi, $entry_title;
 }
 exit 0;