Skip to content

Commit

Permalink
Merge pull request #19 from hubmapconsortium/test-release
Browse files Browse the repository at this point in the history
v2.0.0 release
  • Loading branch information
yuanzhou authored Mar 15, 2021
2 parents 3b78998 + d67341b commit 4092a07
Show file tree
Hide file tree
Showing 20 changed files with 1,466 additions and 674 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Ignore config files
src/instance/app.cfg
app.cfg
reload_from_neo4j/reload.properties

BUILD
# mounted BUILD/VERSION file
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# uuid-api for HuBMAP
# HuBMAP UUID API
The uuid-api service is a restful web service used to create and query UUIDs used across HuBMAP. Three types of IDs are supported:
* HuBMAP UUID: Standard randomly generated 128 bit UUIDs represented as 32 hexadecimal digits. These are generated by the service.
* HuBMAP DOI prefix: A randomly generated unique id that can be used to construct a HuBMAP DOI in the format ###.XXXX.###. These are optionally generated by the service.
* HuBMAP Display Id: An id specified when generating a UUID and stored by the service to associate user defined ids with UUIDs.


## Development and deployment environments
## Docker development and deployment environments

We have the following 5 development and deployment environments:

Expand Down Expand Up @@ -89,4 +89,4 @@ You can also stop the running container and remove it by:

### Updating API Documentation

The documentation for the API calls is hosted on SmartAPI. Modifying the `entity-api-spec.yaml` file and commititng the changes to github should update the API shown on SmartAPI. SmartAPI allows users to register API documents. The documentation is associated with this github account: [email protected]. Please contact Chuck Borromeo ([email protected]) if you want to register a new API on SmartAPI.
The documentation for the API calls is hosted on SmartAPI. Modifying the `entity-api-spec.yaml` file and commititng the changes to github should update the API shown on SmartAPI. SmartAPI allows users to register API documents. The documentation is associated with this github account: [email protected].
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.7.0
2.0.0
4 changes: 2 additions & 2 deletions docker/docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ services:
restart: always
volumes:
# Mount the VERSION file and BUILD file
- "../VERSION:/usr/src/app/src/VERSION"
- "../BUILD:/usr/src/app/src/BUILD"
- "../VERSION:/usr/src/app/VERSION"
- "../BUILD:/usr/src/app/BUILD"
# Mount the source code to container
- "../src:/usr/src/app/src"

32 changes: 16 additions & 16 deletions docker/docker-compose.localhost.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,25 @@ services:
- HOST_UID=${HOST_UID:-1000}
volumes:
# Mount the VERSION file and BUILD file
- "../VERSION:/usr/src/app/src/VERSION"
- "../BUILD:/usr/src/app/src/BUILD"
- "../VERSION:/usr/src/app/VERSION"
- "../BUILD:/usr/src/app/BUILD"
# Mount the source code to container
- "../src:/usr/src/app/src"

# Only used by local development
hubmap-mysql:
build: ./hubmap-mysql
# Build the image with name and tag
image: hubmap-mysql:1.1
hostname: hubmap-mysql
container_name: hubmap-mysql
environment:
MYSQL_ROOT_PASSWORD: 123
# Use the same port mapping for dev and prod
ports:
- "3306:3306"
networks:
- gateway_hubmap
# # Only used by local development
# hubmap-mysql:
# build: ./hubmap-mysql
# # Build the image with name and tag
# image: hubmap-mysql:1.1
# hostname: hubmap-mysql
# container_name: hubmap-mysql
# environment:
# MYSQL_ROOT_PASSWORD: 123
# # Use the same port mapping for dev and prod
# ports:
# - "3306:3306"
# networks:
# - gateway_hubmap



16 changes: 16 additions & 0 deletions docker/docker-compose.refactor.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
version: "3.7"

services:

uuid-api:
environment:
- HOST_GID=${HOST_GID:-1000}
- HOST_UID=${HOST_UID:-1000}
init: true
restart: always
volumes:
# Mount the VERSION file and BUILD file
- "../VERSION:/usr/src/app/VERSION"
- "../BUILD:/usr/src/app/BUILD"
# Mount the source code to container
- "../src:/usr/src/app/src"
2 changes: 1 addition & 1 deletion docker/hubmap-mysql/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ LABEL description="HuBMAP MySQL 5.6 for UUID API Service" \
version="1.1"

# Copy from host to image
COPY ./uuids-dev.sql /docker-entrypoint-initdb.d/uuids-dev.sql
COPY ./uuids.sql /docker-entrypoint-initdb.d/uuids.sql
File renamed without changes.
26 changes: 10 additions & 16 deletions docker/uuid-api-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ function export_version() {
echo "UUID_API_VERSION: $UUID_API_VERSION"
}

if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "test" && "$1" != "stage" && "$1" != "prod" ]]; then
echo "Unknown build environment '$1', specify one of the following: localhost|dev|test|stage|prod"
if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "test" && "$1" != "stage" && "$1" != "prod" && "$1" != "refactor" ]]; then
echo "Unknown build environment '$1', specify one of the following: localhost|dev|test|stage|prod|refactor"
else
if [[ "$2" != "check" && "$2" != "config" && "$2" != "build" && "$2" != "start" && "$2" != "stop" && "$2" != "down" ]]; then
echo "Unknown command '$2', specify one of the following: check|config|build|start|stop|down"
Expand Down Expand Up @@ -95,27 +95,21 @@ else
# Copy over the source code to docker directory
cp -r ../src uuid-api/

# Also load the sample MySQL database for localhost build
if [ "$1" = "localhost" ]; then
# Copy over the sql database
cp -r ../sql/uuids-dev.sql hubmap-mysql/uuids-dev.sql
fi

# Only mount the VERSION file and BUILD file for localhost and dev
# On test/stage/prod, copy the VERSION file and BUILD file to image
if [[ "$1" != "localhost" && "$1" != "dev" ]]; then
if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "refactor" ]]; then
# Delete old VERSION and BUILD files if found
if [ -f "uuid-api/src/VERSION" ]; then
rm -rf uuid-api/src/VERSION
if [ -f "uuid-api/VERSION" ]; then
rm -rf uuid-api/VERSION
fi

if [ -f "uuid-api/src/BUILD" ]; then
rm -rf uuid-api/src/BUILD
if [ -f "uuid-api/BUILD" ]; then
rm -rf uuid-api/BUILD
fi

# Copy over the one files
cp ../VERSION uuid-api/src
cp ../BUILD uuid-api/src
# Copy over the BUILD and VERSION files
cp ../VERSION uuid-api
cp ../BUILD uuid-api
fi

docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p uuid-api build
Expand Down
1 change: 1 addition & 0 deletions reload_from_neo4j/fix-organs.cypher
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
match (s:Sample) where s.organ starts with 'LY' set s.organ = 'LY'
133 changes: 133 additions & 0 deletions reload_from_neo4j/fix_sample_file_uploads.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
import traceback
from py2neo import Graph
from hubmap_commons.exceptions import ErrorMessage
from hubmap_commons.properties import PropHelper
from hubmap_commons import file_helper
from uuid_worker import UUIDWorker
import sys
import os
import json
import hashlib
import requests
from shutil import copyfile

SAMPLES_TO_FIX_Q = "match (s:Sample) where s.portal_metadata_upload_files is not null and not s.portal_metadata_upload_files = '[]' return s.uuid, s.portal_metadata_upload_files"

class FixFileUploads:

def __init__(self, property_file_name):
self.props = PropHelper(property_file_name, required_props = ['neo4j.server', 'neo4j.username', 'neo4j.password', 'db.host', 'db.name', 'db.username', 'db.password', 'user.mapping', 'client.id', 'client.secret', 'old.ingest.upload.dir', 'new.ingest.upload.dir', 'uuid.api.url', 'user.nexus.token'])
self.graph = Graph(self.props.get('neo4j.server'), auth=(self.props.get('neo4j.username'), self.props.get('neo4j.password')))
self.uuid_wrker = UUIDWorker(self.props.get('client.id'), self.props.get('client.secret'), self.props.get('db.host'), self.props.get('db.name'), self.props.get('db.username'), self.props.get('db.password'))
self.old_ingest_upload_dir = file_helper.ensureTrailingSlash(self.props.get('old.ingest.upload.dir'))
self.new_ingest_upload_dir = file_helper.ensureTrailingSlash(self.props.get('new.ingest.upload.dir'))
self.uuid_api_url = file_helper.ensureTrailingSlashURL(self.props.get('uuid.api.url')) + "hmuuid"
self.user_token = self.props.get('user.nexus.token')

def fix_sample_file_uploads(self):
samples = self.graph.run(SAMPLES_TO_FIX_Q).data()
tofix = []
for sample in samples:
uuid = sample['s.uuid']
file_json = sample['s.portal_metadata_upload_files'].replace("'", '"')
file_info = json.loads(file_json)
changes = {"uuid":uuid, "changes":[]}
tofix.append(changes)
for fi in file_info:
change_info = {}
file_path = fi['filepath']
dirs = file_path.split('/')
n_dirs = len(dirs)
old_rel_dir = dirs[n_dirs-3] + os.sep + dirs[n_dirs-2]

change_info['copy_from'] = self.old_ingest_upload_dir + old_rel_dir + os.sep + dirs[n_dirs-1]
if not os.path.exists(change_info['copy_from']):
raise Exception(f'File {change_info["copy_from"]} attached to entity with uuid:{uuid} does not exist')
change_info['file_uuid'] = self.__gen_file_uuid(change_info['copy_from'], uuid + os.sep + '<uuid>' + os.sep + dirs[n_dirs-1], uuid)
new_rel_dir = uuid + os.sep + change_info['file_uuid']
new_rel_path = new_rel_dir + os.sep + dirs[n_dirs-1]
change_info['copy_to'] = self.new_ingest_upload_dir + new_rel_path
change_info['file_path'] = new_rel_path
change_info['file_name'] = dirs[n_dirs-1]
if not os.path.exists(self.new_ingest_upload_dir + new_rel_dir):
os.makedirs(self.new_ingest_upload_dir + new_rel_dir)
description = ""
if 'description' in fi:
change_info['description'] = fi['description']
changes['changes'].append(change_info)

for fix_rcd in tofix:
uuid = fix_rcd['uuid']
new_files_recd = []
for change in fix_rcd['changes']:
print(f'copy {change["copy_from"]} to {change["copy_to"]}')
file_rcd = {}
file_rcd['filename'] = change['file_name']
file_rcd['file_uuid'] = change['file_uuid']
if 'description' in change:
file_rcd['description'] = change['description']
new_files_recd.append(file_rcd)
copyfile(change["copy_from"], change["copy_to"])

#update the record
update_cql = "match (s:Sample {uuid:'" + uuid + "'}) set s.metadata_files = '" + json.dumps(new_files_recd) + "' return s.uuid"
self.graph.run(update_cql)
#print(uuid + ":" + json.dumps(new_files_recd))
#if file_path
#print(f"{uuid}:{file_json}")

def __gen_file_uuid(self, file_current_path, file_rel_path, parent_entity_uuid):
checksum = hashlib.md5(open(file_current_path, 'rb').read()).hexdigest()
filesize = os.path.getsize(file_current_path)
headers = {'Authorization': 'Bearer ' + self.user_token, 'Content-Type': 'application/json'}
data = {}
data['entity_type'] = 'FILE'
data['parent_ids'] = [parent_entity_uuid]
file_info= {}
file_info['path'] = file_rel_path
file_info['checksum'] = checksum
file_info['base_dir'] = 'INGEST_PORTAL_UPLOAD'
file_info['size'] = filesize
data['file_info'] = [file_info]
response = requests.post(self.uuid_api_url, json = data, headers = headers, verify = False)
if response is None or response.status_code != 200:
raise Exception(f"Unable to generate uuid for file {file_rel_path}")

rsjs = response.json()
file_uuid = rsjs[0]['uuid']
return file_uuid


def tfix(self):
samples = self.graph.run("match (s:Sample) where not s.metadata_files is null return s.uuid, s.metadata_files").data()
for sample in samples:
uuid = sample['s.uuid']
file_json = sample['s.metadata_files'].replace("'", '"')
file_info_arry = json.loads(file_json)
replace_arry = []
for file_info in file_info_arry:
if 'file_path' in file_info:
f_path = file_info['file_path']
s_path = f_path.split('/')
file_info['filename'] = s_path[3]
print(file_info['filename'])
file_info.pop('file_path')
replace_arry.append(file_info)
update_cql = "match (s:Sample {uuid:'" + uuid + "'}) set s.metadata_files = '" + json.dumps(replace_arry) + "' return s.uuid"
self.graph.run(update_cql)

try:
fixer = FixFileUploads(os.path.dirname(os.path.realpath(__file__)) + "/reload.properties")
fixer.fix_sample_file_uploads()

except ErrorMessage as em:
print(em.get_message())
exit(1)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
eMsg = str(e)
print("ERROR Occurred: " + eMsg)
traceback.print_tb(exc_traceback)
exit(1)


23 changes: 23 additions & 0 deletions reload_from_neo4j/reload.properties.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Point to the MySQL db that will be loaded
db.host=hubmap-mysql
db.name=hm.uuid
db.username=root
db.password=123

#a mapping of user email addresses to user ids used when
#a user id is missing in neo4j
user.mapping={"[email protected]":"43e53b4a-7853-33e4-99f3-bce224a0e312","[email protected]":"5eab9484-6bec-486d-88cc-7492620a3d6c","[email protected]":"6ce3090a22-11c8-3544-cc53-b91f8d257362"}

#the location of the files previously upload via the ingest portal
old.ingest.upload.dir=/Users/bill/projects/hubmap/ingest-uploads-testing/uploads
#location where the previously uploaded files should be copied
new.ingest.upload.dir=/Users/bill/projects/hubmap/file_uploads

uuid.api.url=https://uuid-api.refactor.hubmapconsortium.org/


# Point to the Neo4j database that will be used
# to load the uuid database
neo4j.server=bolt://hubmap-neo4j:7687
neo4j.username=neo4j
neo4j.password=123
Loading

0 comments on commit 4092a07

Please sign in to comment.