pulling cleasing regexes out to extract, modding multiple spaces to n…

…ot be \s as includes line breaks
dbashford · Aug 24, 2014 · 3693058 · 3693058
1 parent bbb008e
commit 3693058
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 8 deletions.
diff --git a/lib/extract.js b/lib/extract.js
@@ -3,8 +3,11 @@ var fs = require( 'fs' )
   , extractorPath = path.join( __dirname, "extractors" )
   , typeExtractors = {}
   , regexExtractors = []
-  , preserveLineBreaks = /[^A-Za-z\x80-\xFF 0-9 \u2026 \u4E00-\u9FFF \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~'-\w\n\r]*/g
-  , stripLineBreaks = /[^A-Za-z\x80-\xFF 0-9 \u2026 \u4E00-\u9FFF \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~'-\w]*/g
+  , WHITELIST_PRESERVE_LINEBREAKS = /[^A-Za-z\x80-\xFF 0-9 \u2026 \u4E00-\u9FFF \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~'-\w\n\r]*/g
+  , WHITELIST_STRIP_LINEBREAKS = /[^A-Za-z\x80-\xFF 0-9 \u2026 \u4E00-\u9FFF \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~'-\w]*/g
+  , SINGLE_QUOTES = /[\u2018|\u2019]/g
+  , DOUBLE_QUOTES = /[\u201C|\u201D]/g
+  , MULTIPLE_SPACES = /[ \t\v\u00A0]{2,}/g // multiple spaces, tabs, vertical tabs, non-breaking space
   , extractors = []
   , totalExtractors = 0
   , satisfiedExtractors = 0;
@@ -36,12 +39,15 @@ var cleanseText = function( options, cb ) {
     if ( !error ) {
       // clean up text
       if ( options.preserveLineBreaks ) {
-        text = text.replace( preserveLineBreaks, ' ' );
+        text = text.replace( WHITELIST_PRESERVE_LINEBREAKS, ' ' );
       } else {
-        text = text.replace( stripLineBreaks, ' ' );
+        text = text.replace( WHITELIST_STRIP_LINEBREAKS, ' ' );
       }
-      text = text.replace( / (?! )/g, '' );
-      text = text.replace( /\s{2,}/g, ' ' );
+
+      text = text.replace( / (?! )/g, '' )
+        .replace( MULTIPLE_SPACES, ' ' )  // replace 2 or more spaces with 1 space
+        .replace( SINGLE_QUOTES, "'" ) // replace nasty quotes with simple ones
+        .replace( DOUBLE_QUOTES, '"' ); // replace nasty quotes with simple ones
     }
     cb( error, text );
   };

diff --git a/test/extract_test.js b/test/extract_test.js
@@ -159,7 +159,6 @@ describe('textract', function() {
       textract(filePath, function( error, text ) {
         expect(error).to.be.null;
         expect(text).to.be.a('string');
-        console.log(text)
         expect(text).to.eql( "this is a test document that won't be extracted properly." );
         done();
       });
@@ -276,7 +275,7 @@ describe('textract', function() {
       textract(filePath,  {preserveLineBreaks:true}, function( error, text ) {
         expect(error).to.be.null;
         expect(text).to.be.an('string');
-        expect(text.indexOf("…")).to.eql(923);
+        expect(text.indexOf("…")).to.eql(928);
         done();
       });
     });