New CommonData Reader

.github/workflows/fitbot.yml

@@ -10,7 +10,7 @@ on:
 env:
   N3FIT_MAXNREP: 20 # total number of replicas to fit
   POSTFIT_NREP: 16 # requested replicas for postfit
-  REFERENCE_SET: NNBOT-3e7c84220-2024-02-28 # reference set for exact results
+  REFERENCE_SET: NNBOT-344e6f1a9-2024-03-03 # reference set for exact results
   STABLE_REFERENCE_SET: NNBOT-c0f99b7b3-2024-02-28 # reference set for last tag
   CONDA_PY: 310
   PYTHONHASHSEED: "0"

buildmaster/oldcommondata_porter.py

@@ -94,7 +94,7 @@ def create_uncertainties(df, systype_file, is_default=False, use_multiplicative=
             if info["treatment"] not in ["ADD", "MULT"]:
                 raise ValueError(f"Treatment type: {info['treatment']} not recognized")
             if use_multiplicative:
-                tmp[key] = float(bin_data[2 * n+1]*data[idx-1]/100.0)
+                tmp[key] = float(bin_data[2 * n + 1] * data[idx - 1] / 100.0)
             else:
                 tmp[key] = float(bin_data[2 * n])
         bins.append(tmp)
@@ -340,16 +340,9 @@ def convert_from_old_to_new(dsname, new_info, overwrite=False, dry=False, keep_e
         metadata = safe_load(metadata_path.read_text())
         # Perform sanity checks
         nnpdf_md = metadata["nnpdf_metadata"]
-        try:
-            assert nnpdf_md["experiment"] == plotting_dict["experiment"]
-            assert nnpdf_md["nnpdf31_process"] == plotting_dict["nnpdf31_process"]
-            assert metadata.get("setname") == set_name
-        except AssertionError:
-            print(traceback.format_exc())
-            # If this fails, inspect
-            import ipdb
-
-            ipdb.set_trace()
+        assert nnpdf_md["experiment"] == plotting_dict["experiment"]
+        assert nnpdf_md["nnpdf31_process"] == plotting_dict["nnpdf31_process"]
+        assert metadata.get("setname") == set_name
         # Check whether the observable already exists
         already_implemented = [i["observable_name"] for i in metadata["implemented_observables"]]
         if obs_name in already_implemented:

doc/sphinx/source/tutorials/closuretest.md

@@ -171,13 +171,10 @@ the closure test settings modified as shown
 between running a closure fit in ``n3fit`` and a standard fit is that the user is
 required to run ``vp-setupfit`` on the runcard before running ``n3fit``. This is
 because the filtering of the data is required to generate the pseudodata central
-values. The filtered data should then be rebuilt before the fit, so there is no
-risk of the fit crashing due to multiple replicas rebuilding the data
-simultaneously. The workflow is as follows:
+values. The workflow is as follows:
 
 ```bash
 $ vp-setupfit fitname.yml
-$ vp-rebuild-data fitname
 $ n3fit fitname.yml <replica_number>
 ```
 

extra_tests/regression_fits/central.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -31,7 +31,7 @@ theory:
 genrep: False    # on = generate MC replicas, False = use real data
 trvlseed: 3
 nnseed: 2
-mcseed: 1  
+mcseed: 1
 
 load: "weights.h5"
   #save: "weights.h5"
@@ -69,12 +69,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/diagonal.yml

@@ -12,9 +12,9 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
 
 ############################################################
 datacuts:
@@ -69,12 +69,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/feature_scaling.yml

@@ -12,9 +12,9 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -30,7 +30,7 @@ theory:
 genrep: True    # on = generate MC replicas, False = use real data
 trvlseed: 3
 nnseed: 2
-mcseed: 1  
+mcseed: 1
 
 load: "weights_feature.h5"
 
@@ -68,12 +68,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/flavour.yml

@@ -12,8 +12,8 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
 
 ############################################################
 datacuts:
@@ -29,7 +29,7 @@ theory:
 genrep: True    # on = generate MC replicas, False = use real data
 trvlseed: 3
 nnseed: 2
-mcseed: 1  
+mcseed: 1
 
 load: "weights_flavour.h5"
 
@@ -67,12 +67,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/no_lagrange.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -31,7 +31,7 @@ theory:
 genrep: False    # on = generate MC replicas, False = use real data
 trvlseed: 3
 nnseed: 2
-mcseed: 1  
+mcseed: 1
 
 load: "weights.h5"
   #save: "weights.h5"

extra_tests/regression_fits/no_msr.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -70,12 +70,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/no_sumrules.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -70,12 +70,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/no_vsr.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -70,12 +70,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/normal_fit.yml

@@ -6,8 +6,8 @@ description: n3fit regression test, normal fit without initial weights
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
 
 ############################################################
 datacuts:
@@ -23,7 +23,7 @@ theory:
 genrep: True    # on = generate MC replicas, False = use real data
 trvlseed: 3
 nnseed: 2
-mcseed: 1  
+mcseed: 1
 
 save: "weights.h5"
 
@@ -60,12 +60,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/trainable_prepro.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -31,7 +31,7 @@ theory:
 genrep: True    # on = generate MC replicas, False = use real data
 trvlseed: 3
 nnseed: 2
-mcseed: 1  
+mcseed: 1
 
 load: "weights.h5"
   #save: "weights.h5"
@@ -69,12 +69,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true