From 0f8933b9a288adaa0d3fc7c2720875c6ed7fa537 Mon Sep 17 00:00:00 2001 From: Kyle Zollo Date: Wed, 25 Oct 2023 18:32:32 -0400 Subject: [PATCH] Make initializeDatabase call the correct DDL file (#266) --- R/dbUtils.R | 6 +- inst/csv/data_source.csv | 2 - inst/csv/gaia001fieldLevel.csv | 2 +- inst/ddl/001/gaiadb_001_ddl.sql | 2 +- inst/ddl/001/gaiadb_001_sequences.sql | 43 +++++++ inst/sql/backbone_ddl.sql | 167 -------------------------- 6 files changed, 50 insertions(+), 172 deletions(-) create mode 100644 inst/ddl/001/gaiadb_001_sequences.sql delete mode 100644 inst/sql/backbone_ddl.sql diff --git a/R/dbUtils.R b/R/dbUtils.R index 9a7d7cdf..9e5e69e6 100644 --- a/R/dbUtils.R +++ b/R/dbUtils.R @@ -161,7 +161,11 @@ initializeDatabase <- function(connectionDetails, overwrite = FALSE, testing = F return() } } - DatabaseConnector::executeSql(conn, sql = readr::read_file(system.file("sql", "backbone_ddl.sql", package="gaiaCore"))) + DatabaseConnector::executeSql(conn, sql = "CREATE SCHEMA IF NOT EXISTS backbone AUTHORIZATION postgres;") + # TODO the sequences file is not autogenerated. Add it to the create/writeDdl script? + DatabaseConnector::executeSql(conn, sql = readr::read_file(system.file("ddl", "001", "gaiadb_001_sequences.sql", package="gaiaCore"))) + DatabaseConnector::executeSql(conn, sql = readr::read_file(system.file("ddl", "001", "gaiadb_001_ddl.sql", package="gaiaCore"))) + # DatabaseConnector::executeSql(conn, sql = readr::read_file(system.file("sql", "backbone_ddl.sql", package="gaiaCore"))) message("backbone schema created.") if (!testing) { dataSource <- readr::read_csv(system.file(file.path("csv", paste0("data_source.csv")), package = 'gaiaCore')) diff --git a/inst/csv/data_source.csv b/inst/csv/data_source.csv index 7680ecaf..da132bb7 100644 --- a/inst/csv/data_source.csv +++ b/inst/csv/data_source.csv @@ -11,9 +11,7 @@ data_source_uuid,org_id,org_set_id,dataset_name,dataset_version,geom_type,geom_s 8888,EPA,AQS,annual_conc_by_cbsa_2018,2018,,,cbsa,1,10184,file,zip,csv,annual_aqi_by_cbsa_2018.csv,https://aqs.epa.gov/aqsweb/airdata/annual_aqi_by_cbsa_2018.zip,,https://www.epa.gov/aqs 7777,EPA,AQS,annual_conc_by_monitor_2018,2018,point,,,1,7777,file,zip,csv,annual_conc_by_monitor_2018.csv,https://aqs.epa.gov/aqsweb/airdata/annual_conc_by_monitor_2018.zip,,https://www.epa.gov/aqs 1234,Census,TIGER/Line,us-county-2018,2018,polygon,"{""stage_transform"":[""dplyr::mutate(staged,geom_source_coding='GEOID',geom_source_value=GEOID,geom_name=dplyr::select(sf::st_drop_geometry(staged), n = if('NAME' %in% colnames(staged)) 'NAME' else 'NAMELSAD')$n,geom_local_value=geometry,geom_local_epsg=sf::st_crs(staged, paramaters=TRUE)$epsg)""]}",county,0,,file,zip,shp,tl_2018_us_county.shp,https://www2.census.gov/geo/tiger/TIGER2018/COUNTY/tl_2018_us_county.zip,,https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html -6666,WaterQualityData.us,WQX,station,,point,,,0,,API,zip,csv,station.csv,https://www.waterqualitydata.us/data/Station/search?mimeType=csv&zip=yes,,https://www.waterqualitydata.us/webservices_documentation 9999,CDC/ATSDR,SVI,us-county-2018,2018,,,county,1,1234,file,zip,gdb,SVI2018_US_county.gdb,https://svi.cdc.gov/Documents/Data/2018/db/states_counties/SVI_2018_US_county.zip,,https://www.atsdr.cdc.gov/placeandhealth/svi/index.html -5555,WaterQualityData.us,WQX,result,,,,,1,6666,API,zip,csv,result.csv,https://www.waterqualitydata.us/data/Result/search?characterticName={param1}&mimeType=csv&zip=yes,,https://www.waterqualitydata.us/webservices_documentation 10148,Census,TIGER/Line,us-zcta5-2014,2014,polygon,"{""stage_transform"":[""dplyr::mutate(staged,geom_source_coding='GEOID',geom_source_value=GEOID10,geom_local_value=geometry,geom_local_epsg=sf::st_crs(staged, paramaters=TRUE)$epsg)""]}",zcta5,0,,file,zip,shp,tl_2014_us_zcta510.shp,https://www2.census.gov/geo/tiger/TIGER2014/ZCTA5/tl_2014_us_zcta510.zip,,https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html 10159,Census,TIGER/Line,us-zcta5-2015,2015,polygon,"{""stage_transform"":[""dplyr::mutate(staged,geom_source_coding='GEOID',geom_source_value=GEOID10,geom_local_value=geometry,geom_local_epsg=sf::st_crs(staged, paramaters=TRUE)$epsg)""]}",zcta5,0,,file,zip,shp,tl_2015_us_zcta510.shp,https://www2.census.gov/geo/tiger/TIGER2015/ZCTA5/tl_2015_us_zcta510.zip,,https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html 7878,EPA,AQS,daily_aqi_by_county_2018,2018,,,county,1,1234,file,zip,csv,daily_aqi_by_county_2018.csv,https://aqs.epa.gov/aqsweb/airdata/daily_aqi_by_county_2018.zip,,https://www.epa.gov/aqs diff --git a/inst/csv/gaia001fieldLevel.csv b/inst/csv/gaia001fieldLevel.csv index e3001579..5e702232 100644 --- a/inst/csv/gaia001fieldLevel.csv +++ b/inst/csv/gaia001fieldLevel.csv @@ -6,7 +6,7 @@ data_source,dataset_name,Yes,varchar(100),,,No,No,,,,, data_source,dataset_version,Yes,varchar(100),,,No,No,,,,, data_source,geom_type,No,varchar(100),,,No,No,,,,, data_source,geom_spec,No,text,,,No,No,,,,, -data_source,boundary_type,Yes,varchar(100),,,No,No,,,,, +data_source,boundary_type,No,varchar(100),,,No,No,,,,, data_source,has_attributes,No,int4,,,No,No,,,,, data_source,geom_dependency_uuid,No,int4,,,No,Yes,data_source,data_source_uuid,,, data_source,download_method,Yes,varchar(100),,,No,No,,,,, diff --git a/inst/ddl/001/gaiadb_001_ddl.sql b/inst/ddl/001/gaiadb_001_ddl.sql index 58450634..40bdae35 100644 --- a/inst/ddl/001/gaiadb_001_ddl.sql +++ b/inst/ddl/001/gaiadb_001_ddl.sql @@ -7,7 +7,7 @@ CREATE TABLE backbone.data_source ( dataset_version varchar(100) NOT NULL, geom_type varchar(100) NULL, geom_spec text NULL, - boundary_type varchar(100) NOT NULL, + boundary_type varchar(100) NULL, has_attributes int4 NULL, geom_dependency_uuid int4 NULL, download_method varchar(100) NOT NULL, diff --git a/inst/ddl/001/gaiadb_001_sequences.sql b/inst/ddl/001/gaiadb_001_sequences.sql new file mode 100644 index 00000000..e00358b9 --- /dev/null +++ b/inst/ddl/001/gaiadb_001_sequences.sql @@ -0,0 +1,43 @@ +CREATE SEQUENCE IF NOT EXISTS backbone.attr_index_attr_index_id_seq + INCREMENT BY 1 + MINVALUE 1 + MAXVALUE 2147483647 + START 1 + CACHE 1 + NO CYCLE; +-- DROP SEQUENCE backbone.attr_template_attr_record_id_seq; + +CREATE SEQUENCE IF NOT EXISTS backbone.attr_template_attr_record_id_seq + INCREMENT BY 1 + MINVALUE 1 + MAXVALUE 2147483647 + START 1 + CACHE 1 + NO CYCLE; +-- DROP SEQUENCE backbone.variable_source_variable_source_id_seq; + +CREATE SEQUENCE IF NOT EXISTS backbone.variable_source_variable_source_id_seq + INCREMENT BY 1 + MINVALUE 1 + MAXVALUE 2147483647 + START 1 + CACHE 1 + NO CYCLE; +-- DROP SEQUENCE backbone.geom_index_geom_index_id_seq; + +CREATE SEQUENCE IF NOT EXISTS backbone.geom_index_geom_index_id_seq + INCREMENT BY 1 + MINVALUE 1 + MAXVALUE 2147483647 + START 1 + CACHE 1 + NO CYCLE; +-- DROP SEQUENCE backbone.geom_template_geom_record_id_seq; + +CREATE SEQUENCE IF NOT EXISTS backbone.geom_template_geom_record_id_seq + INCREMENT BY 1 + MINVALUE 1 + MAXVALUE 2147483647 + START 1 + CACHE 1 + NO CYCLE;-- backbone.attr_index definition diff --git a/inst/sql/backbone_ddl.sql b/inst/sql/backbone_ddl.sql deleted file mode 100644 index c44674dc..00000000 --- a/inst/sql/backbone_ddl.sql +++ /dev/null @@ -1,167 +0,0 @@ --- DROP SCHEMA backbone; - -CREATE SCHEMA IF NOT EXISTS backbone AUTHORIZATION postgres; - --- DROP SEQUENCE backbone.attr_index_attr_index_id_seq; - -CREATE SEQUENCE IF NOT EXISTS backbone.attr_index_attr_index_id_seq - INCREMENT BY 1 - MINVALUE 1 - MAXVALUE 2147483647 - START 1 - CACHE 1 - NO CYCLE; --- DROP SEQUENCE backbone.attr_template_attr_record_id_seq; - -CREATE SEQUENCE IF NOT EXISTS backbone.attr_template_attr_record_id_seq - INCREMENT BY 1 - MINVALUE 1 - MAXVALUE 2147483647 - START 1 - CACHE 1 - NO CYCLE; --- DROP SEQUENCE backbone.variable_source_variable_source_id_seq; - -CREATE SEQUENCE IF NOT EXISTS backbone.variable_source_variable_source_id_seq - INCREMENT BY 1 - MINVALUE 1 - MAXVALUE 2147483647 - START 1 - CACHE 1 - NO CYCLE; --- DROP SEQUENCE backbone.geom_index_geom_index_id_seq; - -CREATE SEQUENCE IF NOT EXISTS backbone.geom_index_geom_index_id_seq - INCREMENT BY 1 - MINVALUE 1 - MAXVALUE 2147483647 - START 1 - CACHE 1 - NO CYCLE; --- DROP SEQUENCE backbone.geom_template_geom_record_id_seq; - -CREATE SEQUENCE IF NOT EXISTS backbone.geom_template_geom_record_id_seq - INCREMENT BY 1 - MINVALUE 1 - MAXVALUE 2147483647 - START 1 - CACHE 1 - NO CYCLE;-- backbone.attr_index definition - --- Drop table - --- DROP TABLE backbone.attr_index; - -CREATE TABLE IF NOT EXISTS backbone.attr_index ( - attr_index_id serial4 NOT NULL, - attr_of_geom_index_id int4 NULL, - database_schema varchar NULL, - table_name varchar NULL, - data_source_id int4 NULL, - CONSTRAINT attr_index_pkey PRIMARY KEY (attr_index_id) -); - - --- backbone.attr_template definition - --- Drop table - --- DROP TABLE backbone.attr_template; - -CREATE TABLE IF NOT EXISTS backbone.attr_template ( - attr_record_id serial4 NOT NULL, - geom_record_id int4 NULL, - variable_source_record_id int4 NOT NULL, - attr_concept_id int4 NULL, - attr_start_date date NULL, - attr_end_date date NULL, - value_as_number float8 NULL, - value_as_string varchar NULL, - value_as_concept_id int4 NULL, - unit_concept_id int4 NULL, - unit_source_value varchar NULL, - qualifier_concept_id int4 NULL, - qualifier_source_value varchar NULL, - attr_source_concept_id int4 NULL, - attr_source_value varchar NULL, - value_source_value varchar NULL -); - - --- backbone.data_source definition - --- Drop table - --- DROP TABLE backbone.data_source; - -CREATE TABLE IF NOT EXISTS backbone.data_source ( - data_source_uuid int4 NULL, - org_id varchar(100) NULL, - org_set_id varchar(100) NULL, - dataset_name varchar(100) NULL, - dataset_version varchar(100) NULL, - geom_type varchar(100) NULL, - geom_spec text NULL, - boundary_type varchar(100) NULL, - has_attributes int4 NULL, - geom_dependency_uuid int4 NULL, - download_method varchar(100) NULL, - download_subtype varchar(100) NULL, - download_data_standard varchar(100) NULL, - download_filename varchar(100) NULL, - download_url varchar(100) NULL, - download_auth varchar(100) NULL, - documentation_url varchar(100) NULL -); - - --- backbone.variable_source definition - --- Drop table - --- DROP TABLE backbone.variable_source; - -CREATE TABLE IF NOT EXISTS backbone.variable_source ( - variable_source_id serial4 NOT NULL, - variable_name varchar NOT NULL, - variable_desc text NULL, - data_source_uuid int4 NOT NULL, - attr_spec text NULL -); - - --- backbone.geom_index definition - --- Drop table - --- DROP TABLE backbone.geom_index; - -CREATE TABLE IF NOT EXISTS backbone.geom_index ( - geom_index_id int4 NOT NULL GENERATED ALWAYS AS IDENTITY, - data_type_id int4 NULL, - data_type_name varchar NULL, - geom_type_concept_id int4 NULL, - geom_type_source_value varchar NULL, - database_schema varchar NULL, - table_name varchar NULL, - table_desc varchar NULL, - data_source_id int4 NULL, - CONSTRAINT geom_index_pkey PRIMARY KEY (geom_index_id) -); - - --- backbone.geom_template definition - --- Drop table - --- DROP TABLE backbone.geom_template; - -CREATE TABLE IF NOT EXISTS backbone.geom_template ( - geom_record_id serial4 NOT NULL, - geom_name varchar NULL, - geom_source_coding varchar NULL, - geom_source_value varchar NULL, - geom_wgs84 public.geometry NULL, - geom_local_epsg int4 NULL, - geom_local_value public.geometry NULL -);