Merge pull request #587 from NASA-PDS/issue_1

issue #1: command-line option to support content validation for every N products

src/main/java/gov/nasa/pds/tools/label/LocationValidator.java

@@ -429,8 +429,11 @@ public void setRule(String ruleName) {
     this.validationRule = ruleName;
   }
 
+  public void setEveryN(int value) {
+	ruleContext.setEveryN(value);
+  }
   public void setSpotCheckData(int value) {
-    ruleContext.setSpotCheckData(value);
+	ruleContext.setSpotCheckData(value);
   }
 
   public void setAllowUnlabeledFiles(boolean flag) {

src/main/java/gov/nasa/pds/tools/util/EveryNCounter.java

@@ -0,0 +1,18 @@
+package gov.nasa.pds.tools.util;
+
+public class EveryNCounter {
+  private static EveryNCounter myself = null;
+ private static Object lock = new Object();
+  private int current = -1;
+  private EveryNCounter(){
+	}
+  public static EveryNCounter getInstance(){
+	synchronized (lock) {
+      if (myself == null) myself = new EveryNCounter();
+	}
+	return myself;
+  }
+  public int getValue() { return this.current; }
+  public void increment() { this.current++; }
+}
+

src/main/java/gov/nasa/pds/tools/validate/rule/RuleContext.java

@@ -119,6 +119,10 @@ public class RuleContext extends ContextBase {
   /** The key used to indicate whether to disable data content validation. */
   public static final String CHECK_DATA_KEY = "validate.check-data";
 
+  /**
+   * The key used to indicate how many lines or records to skip during content validation.
+   */
+  public static final String EVERY_N_KEY = "validate.every-n";
   /**
    * The key used to indicate how many lines or records to skip during content validation.
    */
@@ -394,12 +398,20 @@ public void setCheckData(boolean flag) {
     putContextValue(CHECK_DATA_KEY, flag);
   }
 
+  public int getEveryN() {
+	return getContextValue(EVERY_N_KEY, Integer.class) == null ? 1 : getContextValue(EVERY_N_KEY, Integer.class);
+  }
+
+  public void setEveryN(int value) {
+	putContextValue(EVERY_N_KEY, value);
+  }
+
   public int getSpotCheckData() {
-    return getContextValue(SPOT_CHECK_DATA_KEY, Integer.class);
+	return getContextValue(SPOT_CHECK_DATA_KEY, Integer.class);
   }
 
   public void setSpotCheckData(int value) {
-    putContextValue(SPOT_CHECK_DATA_KEY, value);
+	  putContextValue(SPOT_CHECK_DATA_KEY, value);
   }
 
   public boolean getAllowUnlabeledFiles() {

src/main/java/gov/nasa/pds/tools/validate/rule/pds4/ArrayValidator.java

@@ -24,6 +24,7 @@
 import gov.nasa.pds.objectAccess.DataType.NumericDataType;
 import gov.nasa.pds.objectAccess.InvalidTableException;
 import gov.nasa.pds.tools.label.ExceptionType;
+import gov.nasa.pds.tools.util.EveryNCounter;
 import gov.nasa.pds.tools.util.FileSizesUtil;
 import gov.nasa.pds.tools.validate.ProblemDefinition;
 import gov.nasa.pds.tools.validate.ProblemListener;
@@ -73,6 +74,8 @@ public boolean validateDataObjectDefinition() {
 
   @Override
   public boolean validateDataObjectContents() throws InvalidTableException, IOException {
+	if (EveryNCounter.getInstance().getValue() % this.context.getEveryN() != 0) return true;
+
     URL target = this.context.getTarget();
     String targetFileName = target.toString().substring(target.toString().lastIndexOf("/") + 1);
     long t0 = System.currentTimeMillis();

src/main/java/gov/nasa/pds/tools/validate/rule/pds4/DataDefinitionAndContentValidationRule.java

@@ -14,6 +14,7 @@
 import gov.nasa.pds.objectAccess.InvalidTableException;
 import gov.nasa.pds.objectAccess.ParseException;
 import gov.nasa.pds.tools.label.ExceptionType;
+import gov.nasa.pds.tools.util.EveryNCounter;
 import gov.nasa.pds.tools.validate.ProblemDefinition;
 import gov.nasa.pds.tools.validate.ProblemType;
 import gov.nasa.pds.tools.validate.ValidationProblem;
@@ -36,10 +37,10 @@ public void validate() throws MalformedURLException, URISyntaxException {
     int objectCounter = 0;
     Label label = null;
     String objectIdentifier = "";
+
     try {
       URL target = getTarget();
       String targetFileName = target.toString().substring(target.toString().lastIndexOf("/") + 1);
-      long t0 = System.currentTimeMillis();
 
       LOG.debug("START definition/content validation: {}", targetFileName);
 
@@ -53,6 +54,7 @@ public void validate() throws MalformedURLException, URISyntaxException {
       DataObjectValidator validator = null;
 
       URL previousDataFile = null;
+      EveryNCounter.getInstance().increment();
       for (DataObject obj : label.getObjects()) {
         objectIdentifier = getObjectIdentifier(obj);
         LOG.debug("Checking DataObject #{} '{}'", objectCounter, obj.getName());

src/main/java/gov/nasa/pds/tools/validate/rule/pds4/TableValidator.java

@@ -32,6 +32,7 @@
 import gov.nasa.pds.objectAccess.table.TableBinaryAdapter;
 import gov.nasa.pds.objectAccess.table.TableDelimitedAdapter;
 import gov.nasa.pds.tools.label.ExceptionType;
+import gov.nasa.pds.tools.util.EveryNCounter;
 import gov.nasa.pds.tools.util.FileService;
 import gov.nasa.pds.tools.util.TableCharacterUtil;
 import gov.nasa.pds.tools.validate.ProblemListener;
@@ -51,6 +52,7 @@
  */
 public class TableValidator implements DataObjectValidator {
   private static final Logger LOG = LoggerFactory.getLogger(TableValidator.class);
+
   private int progressCounter = 0;
   private long currentObjectRecordCounter = 0;
 
@@ -121,6 +123,7 @@ public boolean validateDataObjectDefinition() throws InvalidTableException {
 
   @Override
   public boolean validateDataObjectContents() throws InvalidTableException, IOException, Exception {
+		if (EveryNCounter.getInstance().getValue() % this.context.getEveryN() != 0) return true;
 
     LOG.debug("START table content validation");
     LOG.debug("validateTableDataContents:getTarget() {}", this.context.getTarget());

src/main/java/gov/nasa/pds/validate/ValidateLauncher.java

@@ -220,6 +220,8 @@ public class ValidateLauncher {
 
   private long maxErrors;
 
+  private int everyN;
+
   private int spotCheckData;
 
   private boolean allowUnlabeledFiles;
@@ -270,6 +272,7 @@ public ValidateLauncher() throws TransformerConfigurationException {
     contextReferenceCheck = true;
     skipProductValidation = false;
     maxErrors = MAX_ERRORS;
+    everyN = 1;
     spotCheckData = -1;
     allowUnlabeledFiles = false;
     registeredAndNonRegistedProducts = new HashMap<>();
@@ -319,6 +322,16 @@ public void query(CommandLine line) throws Exception {
     // Initialize flags in FlagsUtil to their default states.
     FlagsUtil.initialize();
 
+    try {
+      setEveryN(Integer.valueOf(line.getOptionValue("everyN", "1")).intValue());
+      if (this.everyN < 1)
+        throw new InvalidOptionException(
+          "Value must be greater than or equal to 1 not '" + line.getOptionValue("everyN", "1") + "'");
+    } catch (IllegalArgumentException a) {
+      throw new InvalidOptionException(
+        "Could not parse value '" + line.getOptionValue("everyN", "1") + "': " + a.getMessage());
+    }
+
     for (Option o : processedOptions) {
       LOG.debug("query:o.getOpt() {}", o.getOpt());
       LOG.debug("query:o.getLongOpt() {}", o.getLongOpt());
@@ -743,6 +756,9 @@ public void query(File configuration) throws ConfigurationException {
       if (config.containsKey(ConfigKey.MAX_ERRORS)) {
         setMaxErrors(config.getLong(ConfigKey.MAX_ERRORS));
       }
+      if (config.containsKey(ConfigKey.EVERY_N)) {
+          setEveryN(config.getInt(ConfigKey.EVERY_N));
+      }
       if (config.containsKey(ConfigKey.SPOT_CHECK_DATA)) {
         setSpotCheckData(config.getInt(ConfigKey.SPOT_CHECK_DATA));
       }
@@ -1052,9 +1068,13 @@ public void setMaxErrors(long value) {
     this.maxErrors = value;
   }
 
+  public void setEveryN(int value) {
+	    this.everyN = value;
+	  }
+
   public void setSpotCheckData(int value) {
-    this.spotCheckData = value;
-  }
+	    this.spotCheckData = value;
+	  }
 
   public void setAllowUnlabeledFiles(boolean flag) {
     this.allowUnlabeledFiles = flag;
@@ -1283,6 +1303,9 @@ public void setupReport() throws IOException {
     } else {
       report.addParameter("   Product Level Validation      off");
     }
+    if (everyN != 1) {
+      report.addParameter("   Data Every N                  " + everyN);
+    }
     if (spotCheckData != -1) {
       report.addParameter("   Data Spot Check               " + spotCheckData);
     }
@@ -1359,6 +1382,7 @@ public boolean doValidation(Map<URL, String> checksumManifest) throws Exception
         validator.setRecurse(traverse);
         validator.setCheckData(contentValidationFlag);
         validator.setSpotCheckData(spotCheckData);
+        validator.setEveryN(everyN);
         validator.setAllowUnlabeledFiles(allowUnlabeledFiles);
         validator.setValidateContext(validateContext);
         validator.setSkipProductValidation(skipProductValidation);

src/main/java/gov/nasa/pds/validate/commandline/options/ConfigKey.java

@@ -111,6 +111,11 @@ public class ConfigKey {
    */
   public static final String MAX_ERRORS = "validate.maxErrors";
 
+  /**
+   * Property to specify how many lines or records to skip during content validation.
+   */
+  public static final String EVERY_N = "validate.everyN";
+
   /**
    * Property to specify how many lines or records to skip during content validation.
    */

src/main/java/gov/nasa/pds/validate/commandline/options/Flag.java

@@ -66,6 +66,7 @@ public enum Flag {
   EXTENSION("e", "label-extension", "xml|lblx", String.class, true,
       "Specify file extension for the labels files. Default: xml. NOTE: Support for intermingled bundles where collections have differing label file extensions is not yet supported."),
 
+  EVERY_N(null, "everyN", "value", int.class, "Process every N files with the default being every file."),
   /**
    * DEPRECATED: Flag to force the tool to perform validation against the schema and schematron
    * specified in a given label.

src/main/java/gov/nasa/pds/validate/commandline/options/FlagOptions.java

@@ -52,6 +52,7 @@ public class FlagOptions {
     options.addOption(new ToolsOption(Flag.CONFIG));
     options.addOption(new ToolsOption(Flag.MAX_ERRORS));
     options.addOption(new ToolsOption(Flag.EXTENSION));
+    options.addOption(new ToolsOption(Flag.EVERY_N));
     options.addOption(new ToolsOption(Flag.HELP));
     options.addOption(new ToolsOption(Flag.REPORT));
     options.addOption(new ToolsOption(Flag.TARGET));