add cache system for json plus unitest for it

.gitignore

@@ -24,6 +24,7 @@ exports/
 /lotemplate/unittest/files/content/*.unittest.odt
 /lotemplate/unittest/files/content/*.unittest.pdf
 /venv
+/lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.json
 /output*.*
 .fontconfig/
 docker-compose.override.yml

API/utils.py

@@ -20,12 +20,16 @@
 host='localhost'
 port='200'
 gworkers=0
-def start_soffice(workers):
+scannedjson=''
+def start_soffice(workers,jsondir):
     global gworkers
     global my_lo
+    global scannedjson
+    scannedjson=jsondir
     gworkers=workers
     os.makedirs("uploads", exist_ok=True)
     os.makedirs("exports", exist_ok=True)
+    os.makedirs(scannedjson, exist_ok=True)
     clean_temp_files()
     my_lo=ot.start_multi_office(nb_env=workers)
 
@@ -139,9 +143,9 @@ def save_file(directory: str, f, name: str, error_caught=False) -> Union[tuple[d
 
 
     cnx = connexion()
-
+    global scannedjson
     try:
-        with ot.TemplateFromExt(f"uploads/{directory}/{name}", cnx, True) as temp:
+        with ot.TemplateFromExt(f"uploads/{directory}/{name}", cnx, True,scannedjson) as temp:
             values = temp.variables
     except ot.errors.TemplateError as e:
         delete_file(directory, name)
@@ -162,7 +166,8 @@ def scan_file(directory: str, file: str, error_caught=False) -> Union[tuple[dict
     :return: a json and optionally an int which represent the status code to return
     """
     cnx = connexion()
-    with ot.TemplateFromExt(f"uploads/{directory}/{file}", cnx, True) as temp:
+    global scannedjson
+    with ot.TemplateFromExt(f"uploads/{directory}/{file}", cnx, True,scannedjson) as temp:
             variables = temp.variables
     return {'file': file, 'message': "Successfully scanned", 'variables': variables}
 
@@ -183,8 +188,9 @@ def fill_file(directory: str, file: str, json, error_caught=False) -> Union[tupl
         print("####\nUsing a list of dict is DEPRECATED, you must directly send the dict.")
         print("See documentation.\n#######")
     cnx = connexion()
+    global scannedjson
     try:
-        with ot.TemplateFromExt(f"uploads/{directory}/{file}", cnx, True) as temp:
+        with ot.TemplateFromExt(f"uploads/{directory}/{file}", cnx, True,scannedjson) as temp:
 
             length = len(json)
             is_name_present = type(json.get("name")) is str

app.py

@@ -8,9 +8,11 @@
 import os
 from shutil import copyfile, rmtree
 import pdb
-
+from os.path import isfile, join
+from os import listdir
 from API import utils
 
+import hashlib
 
 
 app = Flask(__name__)
@@ -62,6 +64,13 @@ def directory_route(directory):
                 {'key': 'file'}), 400
         return utils.save_file(directory, f, secure_filename(f.filename))
     elif request.method == 'DELETE':
+        onlyfiles = [f for f in listdir("uploads/"+directory) if isfile(join("uploads/"+directory, f))]
+        json_cache_dir=utils.scannedjson
+        for file in onlyfiles:
+          with open("uploads/"+directory+"/"+file,'rb') as office:
+                cachedjson=json_cache_dir+"/"+(hashlib.md5(office.read()).hexdigest())+'-'+file+".json"
+                if os.path.exists(cachedjson):
+                    os.remove(cachedjson)
         rmtree(f"uploads/{directory}")
         return {'directory': directory, 'message': 'The directory and all his content has been deleted'}
     elif request.method == 'PATCH':
@@ -84,10 +93,11 @@ def directory_route(directory):
 def file_route(directory, file):
     @after_this_request
     def delete_image(response):
-        try:
-            os.remove(file)
-        except Exception as ex:
-            print("Error delete file " + str(file))
+        if request.method == 'POST':
+            try:
+                os.remove(file)
+            except Exception as ex:
+                print("Error delete file " + str(file))
         return response
     if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''):
         return utils.error_sim(
@@ -122,6 +132,10 @@ def delete_image(response):
         file ,response = utils.fill_file(directory, file, request.json)
         return response
     elif request.method == 'DELETE':
+        json_cache_dir=utils.scannedjson
+        with open("uploads/"+directory+"/"+file,'rb') as office:
+                cachedjson=json_cache_dir+"/"+(hashlib.md5(office.read()).hexdigest())+'-'+file+".json"
+        os.remove(cachedjson)
         os.remove(f"uploads/{directory}/{file}")
         return {'directory': directory, 'file': file, 'message': "File successfully deleted"}
 

gunicorn.conf.py

@@ -3,6 +3,7 @@
 
 workers=int(os.environ.get('NB_WORKERS', 4))
 my_lo=[]
+scannedjson='uploads/scannnedjson'
 def on_starting(server):
 
-    utils.start_soffice(workers)
+    utils.start_soffice(workers,scannedjson)

lotemplate/Template.py

@@ -12,7 +12,7 @@
 import os
 from typing import Union
 from sorcery import dict_of
-
+import hashlib
 import uno
 import unohelper
 from com.sun.star.beans import PropertyValue
@@ -24,7 +24,7 @@
 from com.sun.star.style.BreakType import PAGE_AFTER
 
 from . import errors
-from . import Connexion 
+from . import Connexion
 from .utils import *
 
 from lotemplate.Statement.ForStatement import ForStatement
@@ -36,6 +36,7 @@
 from lotemplate.Statement.CounterStatement import CounterManager
 import uuid
 import shutil
+import json
 import pdb
 
 
@@ -94,7 +95,7 @@ def open_doc_from_url(self):
     def validDocType(self,doc):
            pass
 
-    def __init__(self, file_path: str, cnx: Connexion, should_scan: bool):
+    def __init__(self, file_path: str, cnx: Connexion, should_scan: bool, json_cache_dir=None):
         """
         An object representing a LibreOffice/OpenOffice template that you can fill, scan, export and more
 
@@ -116,7 +117,20 @@ def __init__(self, file_path: str, cnx: Connexion, should_scan: bool):
             self.variables = None
             self.doc = None
             self.doc = self.open_doc_from_url()
-            self.variables = self.scan(should_close=True) if should_scan else None
+            if json_cache_dir:
+                with open(file_path,'rb') as office:
+                    cachedjson=json_cache_dir+"/"+(hashlib.md5(office.read()).hexdigest())+'-'+self.file_name+".json"
+                if os.path.exists(cachedjson) and should_scan :
+                    try:
+                        with open(cachedjson) as f:
+                            self.variables = json.load(f)
+                        return
+                    except:
+                        pass
+            self.variables = self.scan(should_close=True)
+            if json_cache_dir:
+                with open(cachedjson, 'w') as f:
+                    json.dump(self.variables, f, ensure_ascii=False)
         else:
             raise errors.FileNotFoundError(
                 'file_not_found',

lotemplate/connexion.py

@@ -36,7 +36,6 @@ def start_office(host:str="localhost",port:str="2000"):
     :param port:   define port in the UNO connect-string --accept
     environnement had to be different for each environnement
     """
-
     subprocess.Popen(
              shlex.split('soffice \
              -env:UserInstallation="file:///tmp/LibO_Process'+port+'" \
@@ -77,9 +76,9 @@ def __init__(self, host: str, port: str):
                     "com.sun.star.bridge.UnoUrlResolver", self.local_ctx
                 ).resolve(f"uno:socket,host={host},port={port};urp;StarOffice.ComponentContext")
             except (NoConnectException, RuntimeException) as e:
-                if attempt==1:
-                    sleep(3)
-                elif attempt<2:
+                if attempt==0:
+                    sleep(4)
+                elif  attempt<2:
                     start_office(host,port)
                     sleep(5)
                 else:

lotemplate/lofunction.py

@@ -18,14 +18,14 @@
 import shlex,subprocess
 import random
 import pdb
-def TemplateFromExt(file_path: str, cnx: Connexion, should_scan: bool):
+def TemplateFromExt(file_path: str, cnx: Connexion, should_scan: bool,json_cache_dir=None):
 
         filename, file_extension = os.path.splitext(file_path)
         ods_ext=('.xls','.xlsx','.ods')
         if file_extension in ods_ext:
-             document = CalcTemplate(file_path, cnx , should_scan)
+             document = CalcTemplate(file_path, cnx , should_scan,json_cache_dir)
         else:
-             document = WriterTemplate(file_path, cnx , should_scan)
+             document = WriterTemplate(file_path, cnx , should_scan,json_cache_dir)
         return document
 
 

lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.expected.json

@@ -0,0 +1 @@
+{"TOTO": {"type": "text", "value": ""}, "second": {"type": "text", "value": ""}, "titi": {"type": "text", "value": ""}, "toto": {"type": "text", "value": ""}, "myvar": {"type": "text", "value": ""}, "foobar": {"type": "text", "value": ""}}

lotemplate/unittest/test_cachedjson.py

@@ -0,0 +1,26 @@
+"""
+Copyright (C) 2023 Probesys
+"""
+import unittest
+
+import lotemplate as ot
+from time import sleep
+import subprocess
+import filecmp
+import os
+import json
+from lotemplate.unittest.test_function import *
+
+cnx = start_office()
+
+
+class Test_calc(unittest.TestCase):
+
+    def test_cachedjson(self):
+        cachejson="lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.json"
+        if os.path.isfile(cachejson):
+            os.remove(cachejson)
+        ot.TemplateFromExt("lotemplate/unittest/files/templates/calc_variables.ods",cnx,True,json_cache_dir='lotemplate/unittest/files/content/')
+        self.assertTrue(filecmp.cmp(cachejson,"lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.expected.json", shallow=False))
+
+

lotemplate/unittest/test_template_scan.py

@@ -6,8 +6,7 @@
 import lotemplate as ot
 from time import sleep
 import subprocess
-
-from test_function import *
+from lotemplate.unittest.test_function import *
 
 
 
@@ -93,7 +92,6 @@ class Ifs(unittest.TestCase):
     def test_no_endif(self):
         with self.assertRaises(ot.errors.TemplateError):
             (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_if_statement.odt", cnx, False)).scan()
-        doc.close()
 
 
 class Tables(unittest.TestCase):
@@ -122,12 +120,10 @@ def test_two_row_varied(self):
     def test_invalid_var(self):
         with self.assertRaises(ot.errors.TemplateError):
             (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_var_tab.odt", cnx, False)).scan()
-        doc.close()
 
     def test_invalid_vars(self):
         with self.assertRaises(ot.errors.TemplateError):
             (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_vars_tab.odt", cnx, False)).scan()
-        doc.close()
 
     def test_for(self):
         self.assertEqual(
@@ -139,7 +135,6 @@ def test_for(self):
     def test_for_missing_endfor(self):
         with self.assertRaises(ot.errors.TemplateError):
             (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/for_missing_endfor.odt", cnx, False)).scan()
-        doc.close()
 
     def test_two_tabs_varied(self):
         self.assertEqual(
@@ -220,7 +215,6 @@ def test_invalid_path(self):
     def test_duplicated_variable(self):
         with self.assertRaises(ot.errors.TemplateError):
             (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/duplicated_variables.odt", cnx, False)).scan()
-        doc.close()
 
 
 class OtherFormats(unittest.TestCase):
@@ -315,7 +309,6 @@ def test_for_inside_if(self):
     def test_if_too_many_endif(self):
         with self.assertRaises(ot.errors.TemplateError) as cm:
             (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/if_too_many_endif.odt", cnx, False)).scan()
-            doc.close()
         self.assertEqual(cm.exception.code, "too_many_endif_found")
 
     def test_if_syntax_error(self):

lotemplate_cli.py

@@ -42,6 +42,7 @@ def set_arguments() -> cparse.Namespace:
                    help="Specify if the program should just scan the template and return the information, or fill it.")
     p.add_argument('--force_replacement', '-f', action='store_true',
                    help="Specify if the program should ignore the scan's result")
+    p.add_argument('--json_cache_dir',nargs='?',  help="Specify a cache for the scanned json")
     return p.parse_args()
 
 if __name__ == '__main__':
@@ -60,7 +61,7 @@ def set_arguments() -> cparse.Namespace:
     # establish the connection to the server
     connexion = ot.randomConnexion(my_lo)
     # generate the document to operate and its parameters
-    document = ot.TemplateFromExt(args.template_file, connexion, not args.force_replacement)
+    document = ot.TemplateFromExt(args.template_file, connexion, not args.force_replacement,args.json_cache_dir)
 
     # prints scan result in json format if it should
     if args.scan: