From 079b75fd2a53648daf3ccb18caae7acbf6bc2aca Mon Sep 17 00:00:00 2001 From: Ali Baker Date: Tue, 5 Jan 2021 20:32:31 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=9D=20update=20setup.py=20and=20split?= =?UTF-8?q?=20the=20code=20to=20two=20file=20to=20=20simplify=20developmen?= =?UTF-8?q?t=20process?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Generator/SampleDataInserter.sh | 10 ++-- MANIFEST.in | 5 +- redismirror/__init__.py | 0 redismirror/helper.py | 0 redismirror/main.py | 72 ++++++++++++++++++++++++++++ redismirror/redisCommand.py | 22 +++++++++ run.py | 83 --------------------------------- setup.cfg | 2 + setup.py | 4 +- 9 files changed, 105 insertions(+), 93 deletions(-) create mode 100644 redismirror/__init__.py create mode 100644 redismirror/helper.py create mode 100644 redismirror/main.py create mode 100644 redismirror/redisCommand.py delete mode 100644 run.py diff --git a/Generator/SampleDataInserter.sh b/Generator/SampleDataInserter.sh index 83c272c..a74dddb 100644 --- a/Generator/SampleDataInserter.sh +++ b/Generator/SampleDataInserter.sh @@ -14,11 +14,11 @@ for i in $(seq 1 "${numberOfInsert}"); do redis-cli set "${key1}_${i}_json_ran" "$key4" redis-cli set "${key1}_${i}_image_ran" $key5 - redis-cli hset "${key2}_${i}_uuid_ran" ${i} "$key1" - redis-cli hset "${key2}_${i}_date_ran" ${i} "$key2" - redis-cli hset "${key2}_${i}_date2_ran" ${i} "$key3" - redis-cli hset "${key2}_${i}_json_ran" $i "$key4" - redis-cli hset "${key2}_${i}_image_ran" $i $key5 + redis-cli hset ${key2}_${i}_uuid_ran ${i} "$key1" + redis-cli hset ${key2}_${i}_date_ran ${i} "$key2" + redis-cli hset ${key2}_${i}_date2_ran ${i} "$key3" + redis-cli hset ${key2}_${i}_json_ran $i "$key4" + redis-cli hset ${key2}_${i}_image_ran $i $key5 sleep "$insertSleepTime" echo "$key1 $key2 $key3" done diff --git a/MANIFEST.in b/MANIFEST.in index bba08ef..6e9f61b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ recursive-include *.py include setup.cfg -include README.md CHANGELOG.md LICENSE.md -include *.txt -recursive-include redismirror/templates * +include README.md LICENSE +include *.md diff --git a/redismirror/__init__.py b/redismirror/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/redismirror/helper.py b/redismirror/helper.py new file mode 100644 index 0000000..e69de29 diff --git a/redismirror/main.py b/redismirror/main.py new file mode 100644 index 0000000..f1c7cea --- /dev/null +++ b/redismirror/main.py @@ -0,0 +1,72 @@ +import click +import sys +import redis +import re +from redisCommand import * +# TO DO: +# 1. Add timer to mirror process +# 2. Add auth option +# 3. Support to split it to multi node +# 4. Support all redis command +# 5. Add number of item to mirror limit + + +@click.command() +@click.option("--host", default="127.0.0.1", help="Destination redis host/IP.") +@click.option("--port", default=6379, help="Destination redis port.") +@click.option("--db", default="0", required=False, help="Destination redis DB.") +@click.option("--auth", required=False, help="Destination redis auth info.") +def main(host, port, db, auth): + r = makeConnection(host, port, db, auth) + getSTDOUT(r) + + +def makeConnection(host, port, db, auth): + """Function to create redis connection + + Note: + If the connection failed, the program will exit 1 + + Args: + host (str): redis connection string + port (int): redis port number + db (int): redis database name + auth (str): auth info for redis + + Returns: + connection: connection object for redis + """ + + pool = redis.ConnectionPool(host=host, port=port, db=db) + r = redis.StrictRedis(connection_pool=pool) + try: + r.ping() + print("Redis is connected..") + except Exception as e: + print(f"Redis connection error ({e})") + sys.exit(1) + return r + + +def getSTDOUT(connection): + if not sys.stdin.isatty(): + input_stream = sys.stdin + else: + print("There is no stdin, check help for more info. exit 1") + sys.exit(1) + for line in input_stream: + try: + tmpLine = line.split(" ", 5) + keyType = str(tmpLine[3]).strip('"') + if keyType == "set": + setFunc(line, connection) + elif keyType == "hset": + hsetFunc(line, connection) + else: + print(f"key type ({keyType}) if not supported yet.") + except: + print("Skip line | {0} ....".format(line[:100])) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/redismirror/redisCommand.py b/redismirror/redisCommand.py new file mode 100644 index 0000000..cd11367 --- /dev/null +++ b/redismirror/redisCommand.py @@ -0,0 +1,22 @@ + +def setFunc(rLine, connection): + line = rLine.split(" ", 5) + keyName = line[4].strip('"') + keyValue = line[5].strip('"') + try: + connection.set(keyName, keyValue) + print(f"Key mirroed sucessfully | {keyName}") + except Exception as e: + print(f"Failed to set key | {keyName} | {e}") + + +def hsetFunc(rLine, connection): + line = rLine.split(" ", 6) + keyName = line[4].strip('"') + hkeyName = line[5].strip('"') + keyValue = str(line[6].strip('"')).lstrip('"\n') + try: + connection.hset(keyName, hkeyName, keyValue) + print(f"Key mirroed sucessfully | {keyName}") + except Exception as e: + print(f"Failed to set key | {keyName} | {hkeyName} | {e} ") diff --git a/run.py b/run.py deleted file mode 100644 index c83cfc0..0000000 --- a/run.py +++ /dev/null @@ -1,83 +0,0 @@ -import click -import sys -import redis - -# TO DO: -# 1. Add timer to mirror process -# 2. Add auth option -# 3. Support to split it to multi node -# 4. Support all redis command - - -def makeConnection(connectionString, port, db): - print("Init connection redis..") - pool = redis.ConnectionPool(host=connectionString, port=port, db=db) - r = redis.StrictRedis(connection_pool=pool) - try: - r.ping() - print("Redis is connected..") - except Exception as e: - print( - f"Error in redis, please check the connection info or the redis it self. {e}" - ) - sys.exit(1) - return r - - -def setFunc(rLine, connection): - line = rLine.split(" ", 5) - keyName = line[4].strip('"') - keyValue = line[5].strip('"') - try: - connection.set(keyName, keyValue) - print(f"Key mirroed sucessfully | {keyName}") - except Exception as e: - print(f"Failed to set key | {keyName} | {e}") - - -def hsetFunc(rLine, connection): - line = rLine.split(" ", 6) - keyName = line[4].strip('"') - hkeyName = line[5].strip('"') - keyValue = str(line[6].strip('"')).lstrip('"\n') - try: - connection.hset(keyName, hkeyName, keyValue) - print(f"Key mirroed sucessfully | {keyName}") - except Exception as e: - print(f"Failed to set key | {keyName} | {hkeyName} | {e} ") - - -def getSTDOUT(connection): - if not sys.stdin.isatty(): - input_stream = sys.stdin - else: - print("There is no stdin, check help for more info. exit 1") - sys.exit(1) - for line in input_stream: - try: - tmpLine = line.split(" ", 5) - keyType = str(tmpLine[3]).strip('"') - if keyType == "set": - setFunc(line, connection) - elif keyType == "hset": - hsetFunc(line, connection) - else: - print(f"key type ({keyType}) if not supported yet.") - except: - print(f"Skip line | {line}") - - -@click.command() -@click.option( - "--host", default="127.0.0.1", help="Location of media file to be converted" -) -@click.option("--port", default=6379, help="Location of media file to be converted") -@click.option("--db", default="0", help="Location of media file to be converted") -@click.option("--auth", required=False, help="Location of media file to be converted") -def main(host, port, db, auth): - rd = makeConnection(host, port, db) - getSTDOUT(rd) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index e69de29..224a779 100644 --- a/setup.cfg +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index 51e8f62..8cf1b74 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ from setuptools import setup, find_packages -VERSION = get_version() f = open("README.md", "r") LONG_DESCRIPTION = f.read() @@ -27,8 +26,9 @@ packages=find_packages(exclude=["ez_setup", "tests*"]), package_data={"redismirror": ["templates/*"]}, include_package_data=True, + keywords=["traffic", "mirror", "redis", "migration", "cli"], entry_points=""" [console_scripts] - redismirror = run:main + redismirror = redismirror.main:main """, )