vfl LR ckks (intel-analytics#6001)

* vfl LR ckks (intel-analytics#10) * tests * add census * add train * update census * update * add time count * add new layer * update vfl client * add decrypt * ckksTest * Merge ckks in FLServer LR Aggregator (#7) * update ckks in FLServer LR Aggregator * some changes * support ckks at scala FL Server * add * add * fix and add ut * fix and add ut * add support for customized client module and divide example to 2 parties * add comments * split * Ckks & Dllib Example fix (#8) * update ckks in FLServer LR Aggregator * some changes * support ckks at scala FL Server * add * add * fix and add ut * fix and add ut * add support for customized client module and divide example to 2 parties * add comments * fix * fix none target sample * fix model * fix train * fix accuracy * add shuffle * enable ckks * fix ckks * Ckks fix the bug and add server error message (#9) * update ckks in FLServer LR Aggregator * some changes * support ckks at scala FL Server * add * add * fix and add ut * fix and add ut * add support for customized client module and divide example to 2 parties * add comments * fix * fix * some fix * update Co-authored-by: Song Jiaming <[email protected]> * remove useless dependency * update client * update client * update * fix style * update * update unit test * code clean up * fix ut * add all-dependencies * update * update readme * update readme and pom * fix ut * update pom * update pom * update * add documents * some update * revert some code Co-authored-by: Song Jiaming <[email protected]>
ForJadeForest · Oct 13, 2022 · b388e75 · b388e75
1 parent e86569c
commit b388e75
Show file tree

Hide file tree

Showing 33 changed files with 2,844 additions and 596 deletions.
diff --git a/scala/assembly/pom.xml b/scala/assembly/pom.xml
@@ -76,7 +76,7 @@
                     <artifactId>spark-mllib_${scala.major.version}</artifactId>
                 </exclusion>
             </exclusions>
-        </dependency>
+      </dependency>
       <dependency>
         <groupId>com.intel.analytics.bigdl</groupId>
         <artifactId>bigdl-serving-spark_${spark.version}</artifactId>
@@ -92,6 +92,64 @@
             </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+          <groupId>org.scala-lang</groupId>
+          <artifactId>scala-library</artifactId>
+          <version>${scala.version}</version>
+          <scope>${spark-scope}</scope>
+      </dependency>
+      <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-core_${scala.major.version}</artifactId>
+          <version>${spark.version}</version>
+          <scope>${spark-scope}</scope>
+          <exclusions>
+              <exclusion>
+                  <groupId>log4j</groupId>
+                  <artifactId>log4j</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>org.slf4j</groupId>
+                  <artifactId>slf4j-log4j12</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>com.fasterxml.jackson.core</groupId>
+                  <artifactId>jackson-databind</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>org.apache.zookeeper</groupId>
+                  <artifactId>zookeeper</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>org.glassfish.jersey.media</groupId>
+                  <artifactId>jersey-media-jaxb</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>org.codehaus.jackson</groupId>
+                  <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>org.apache.commons</groupId>
+                  <artifactId>commons-compress</artifactId>
+              </exclusion>
+          </exclusions>
+      </dependency>
+      <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-mllib_${scala.major.version}</artifactId>
+          <version>${spark.version}</version>
+          <scope>${spark-scope}</scope>
+          <exclusions>
+              <exclusion>
+                  <groupId>org.glassfish.jersey.media</groupId>
+                  <artifactId>jersey-media-jaxb</artifactId>
+              </exclusion>
+              <exclusion>
+                  <groupId>org.codehaus.jackson</groupId>
+                  <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+          </exclusions>
+      </dependency>
     </dependencies>
 
     <build>
@@ -127,6 +185,18 @@
                             </descriptors>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>with-all-in-one</id>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>${project.basedir}/src/main/assembly/all-in-one.xml</descriptor>
+                            </descriptors>
+                        </configuration>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>

diff --git a/scala/assembly/src/main/assembly/all-in-one.xml b/scala/assembly/src/main/assembly/all-in-one.xml
@@ -0,0 +1,23 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+    <id>jar-with-all-dependencies</id>
+    <formats>
+        <format>jar</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>provided</scope>
+        </dependencySet>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/scala/ppml/demo/ppml-conf.yaml b/scala/ppml/demo/ppml-conf.yaml
@@ -3,8 +3,5 @@ clientNum: 2
 #privateKeyFilePath: /ppml/trusted-big-data-ml/work/keys/server.pem
 #certChainFilePath:  /ppml/trusted-big-data-ml/work/keys/server.crt
 serverPort: 8980
-
-##### Client property
-clientTarget: localhost:8980
-# taskID:
+#ckksSercetPath: /ppml/trusted-big-data-ml/work/keys/ckks.crt
 
diff --git a/scala/ppml/scripts/vfl_split_dataset.py b/scala/ppml/scripts/vfl_split_dataset.py
@@ -49,7 +49,7 @@ def vfl_split_dataset(file_name, num_pieces, has_rowkey_index):
         for i, row in enumerate(csv_reader):
             row = np.array(row)
             for j in range(num_pieces):
-                writer_list[j].writerow(row[np.array(col_idx_list[j])])   
+                writer_list[j].writerow(row[np.array(col_idx_list[j])])
 
 
 if __name__ == "__main__":

diff --git a/scala/ppml/src/assembly/all-in-one.xml b/scala/ppml/src/assembly/all-in-one.xml
@@ -0,0 +1,23 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+    <id>jar-with-all-dependencies</id>
+    <formats>
+        <format>jar</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>provided</scope>
+        </dependencySet>
+        <dependencySet>
+            <outputDirectory>/</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <unpack>true</unpack>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/scala/ppml/src/main/java/com/intel/analytics/bigdl/ppml/fl/FLHelper.java b/scala/ppml/src/main/java/com/intel/analytics/bigdl/ppml/fl/FLHelper.java
@@ -28,6 +28,7 @@ public class FLHelper {
     String certChainFilePath = null;
     String privateKeyFilePath = null;
 
+    String ckksSercetPath = "";
     String fgBoostServerModelPath = null;
 
     public void setPsiSalt(String psiSalt) { this.psiSalt = psiSalt; }
@@ -51,6 +52,10 @@ public void setTaskID(String taskID) {
     public void setCertChainFilePath(String certChainFilePath) {
        this.certChainFilePath = certChainFilePath;
     }
+
+    public void setCkksSercetPath(String ckksSercetPath) {
+        this.ckksSercetPath = ckksSercetPath;
+    }
 
     public void setPrivateKeyFilePath(String privateKeyFilePath) {
        this.privateKeyFilePath = privateKeyFilePath;

diff --git a/scala/ppml/src/main/java/com/intel/analytics/bigdl/ppml/fl/common/Aggregator.java b/scala/ppml/src/main/java/com/intel/analytics/bigdl/ppml/fl/common/Aggregator.java
@@ -71,6 +71,7 @@ public void putClientData(FLPhase flPhase, int clientUUID, int version, DataHold
             if (storageHolder.getClientDataSize() >= clientNum) {
                 logger.debug("Server start aggregate: " + flPhase);
                 aggregate(flPhase);
+                logger.debug("Server aggregate finished: " + flPhase);
                 notifyAll();
             } else {
                 wait();