From d60b8f56252dc384c3f71bec3f50300f8afbefc5 Mon Sep 17 00:00:00 2001 From: Jim Nasby Date: Thu, 7 Nov 2019 08:50:59 -0600 Subject: [PATCH] WIP: Add support for testing upgrade scripts (#128) Add support for testing of pgTap update scripts. This commit adds several new make targets: - make uninstall-all: remove ALL installed pgtap code. Unlike `make unintall`, this removes pgtap*, not just our defined targets. Useful when testing multiple versions of pgtap. - make regress: run installcheck then print any diffs from expected output. - make updatecheck: install an older version of pgTap from PGXN (controlled by $UPDATE_FROM; 0.95.0 by default), update to the current version via ALTER EXTENSION, then run installcheck. - make results: runs `make test` and copies all result files to test/expected/. DO NOT RUN THIS UNLESS YOU'RE CERTAIN ALL YOUR TESTS ARE PASSING! In addition to these changes, `make installcheck` now runs as many tests as possible in parallel. This is much faster than running them sequentially. The degree of parallelism can be controlled via `$PARALLEL_CONN`. Setting `$PARALLEL_CONN` to `1` will go back to a serial test schedule. --- .gitignore | 8 +- .travis.yml | 17 +- Makefile | 348 +++++++++++++++++++++++++++---- README.md | 2 +- pg-travis-test.sh | 63 ++++++ sql/pgtap--0.97.0--0.98.0.sql.in | 17 ++ sql/pgtap--0.98.0--0.99.0.sql.in | 20 ++ test/expected/build.out | 1 + test/expected/create.out | 1 + test/expected/extension.out | 6 +- test/expected/runjusttests_6.out | 41 ++++ test/expected/update.out | 4 + test/psql.sql | 14 ++ test/schedule/build.sql | 10 + test/schedule/create.sql | 3 + test/schedule/main.sch | 2 + test/schedule/update.sch | 1 + test/schedule/update.sql | 20 ++ test/setup.sql | 22 +- test/sql/.gitignore | 3 + test/sql/extension.sql | 27 +-- test/sql/performs_ok.sql | 4 +- tools/parallel_conn.sh | 35 ++++ tools/psql_args.sh | 24 +++ 24 files changed, 602 insertions(+), 91 deletions(-) create mode 100644 pg-travis-test.sh create mode 100644 test/expected/build.out create mode 100644 test/expected/create.out create mode 100644 test/expected/runjusttests_6.out create mode 100644 test/expected/update.out create mode 100644 test/psql.sql create mode 100644 test/schedule/build.sql create mode 100644 test/schedule/create.sql create mode 100644 test/schedule/main.sch create mode 100644 test/schedule/update.sch create mode 100644 test/schedule/update.sql create mode 100644 test/sql/.gitignore create mode 100755 tools/parallel_conn.sh create mode 100755 tools/psql_args.sh diff --git a/.gitignore b/.gitignore index 0f572728d000..565d2934311d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ +# THERE IS ANOTHER .gitignore IN test/sql! + .*.swp pgtap.sql pgtap-core.sql pgtap-schema.sql uninstall_pgtap.sql -test/setup.sql results pgtap.so regression.* @@ -11,6 +12,7 @@ regression.* *.html1 *.sql.orig bbin + /sql/pgtap--?.??.?.sql /sql/pgtap--?.?.?.sql /sql/pgtap-core--* @@ -21,6 +23,10 @@ bbin /sql/pgtap--0.98.0--0.99.0.sql /sql/pgtap--0.99.0--1.0.0.sql /sql/pgtap-static.sql +/sql/pgtap-static.sql.tmp* +*.sql.orig + +test/build # Misc mac crap .DS_Store diff --git a/.travis.yml b/.travis.yml index 02ca1e4bee70..85c226e14451 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,12 @@ language: c before_install: - wget https://gist.githubusercontent.com/petere/5893799/raw/apt.postgresql.org.sh - - wget https://gist.githubusercontent.com/petere/6023944/raw/pg-travis-test.sh - sudo sh ./apt.postgresql.org.sh - sudo rm -vf /etc/apt/sources.list.d/pgdg-source.list +script: + - bash ./pg-travis-test.sh + env: - - PGVERSION=8.4 - - PGVERSION=9.0 - PGVERSION=9.1 - PGVERSION=9.2 - PGVERSION=9.3 @@ -14,5 +14,14 @@ env: - PGVERSION=9.5 - PGVERSION=9.6 - PGVERSION=10 - - PGVERSION=11 + - PGVERSION=11 UPDATE_FROM=0.99.0 + # Duplication below is via s/-/- PARALLEL_CONN=1/ + - PARALLEL_CONN=1 PGVERSION=9.1 + - PARALLEL_CONN=1 PGVERSION=9.2 + - PARALLEL_CONN=1 PGVERSION=9.3 + - PARALLEL_CONN=1 PGVERSION=9.4 + - PARALLEL_CONN=1 PGVERSION=9.5 + - PARALLEL_CONN=1 PGVERSION=9.6 + - PARALLEL_CONN=1 PGVERSION=10 + - PARALLEL_CONN=1 PGVERSION=11 UPDATE_FROM=0.99.0 script: bash ./pg-travis-test.sh diff --git a/Makefile b/Makefile index 731a51f15462..51177dfc6267 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,56 @@ TESTS = $(wildcard test/sql/*.sql) EXTRA_CLEAN = $(VERSION_FILES) sql/pgtap.sql sql/uninstall_pgtap.sql sql/pgtap-core.sql sql/pgtap-schema.sql doc/*.html EXTRA_CLEAN += $(wildcard sql/*.orig) # These are files left behind by patch DOCS = doc/pgtap.mmd -REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS)) -REGRESS_OPTS = --inputdir=test --load-language=plpgsql PG_CONFIG ?= pg_config +# +# Test configuration. This must be done BEFORE including PGXS +# + +# If you need to, you can manually pass options to pg_regress with this variable +REGRESS_CONF ?= + +# Set this to 1 to force serial test execution; otherwise it will be determined from Postgres max_connections +PARALLEL_CONN ?= + +# This controls what version to upgrade FROM when running updatecheck. +UPDATE_FROM ?= 0.95.0 + +# +# Setup test variables +# +# We use the contents of test/sql to create a parallel test schedule. Note that +# there is additional test setup below this; some variables must be set before +# loading PGXS, some must be set afterwards. +# + +# These are test files that need to end up in test/sql to make pg_regress +# happy, but these should NOT be treated as regular regression tests +SCHEDULE_TEST_FILES = $(wildcard test/schedule/*.sql) +SCHEDULE_DEST_FILES = $(subst test/schedule,test/sql,$(SCHEDULE_TEST_FILES)) +EXTRA_CLEAN += $(SCHEDULE_DEST_FILES) + +# The actual schedule files. Note that we'll build 2 additional files +SCHEDULE_FILES = $(wildcard test/schedule/*.sch) + +# These are our actual regression tests +TEST_FILES = $(filter-out $(SCHEDULE_DEST_FILES),$(wildcard test/sql/*.sql)) + +# Plain test names +TESTS = $(notdir $(TEST_FILES:.sql=)) + +# Some tests fail when run in parallel +SERIAL_TESTS = coltap hastap + +# This is a bit of a hack, but if REGRESS isn't set we can't installcheck, and +# it must be set BEFORE including pgxs. Note this gets set again below +REGRESS = --schedule $(TB_DIR)/run.sch + +# REMAINING TEST VARIABLES ARE DEFINED IN THE TEST SECTION +# sort is necessary to remove dupes so install won't complain +DATA = $(sort $(wildcard sql/*--*.sql) $(_IN_PATCHED)) # NOTE! This gets reset below! + +# Locate PGXS and pg_config ifdef NO_PGXS top_builddir = ../.. PG_CONFIG := $(top_builddir)/src/bin/pg_config/pg_config @@ -23,10 +69,14 @@ else PGXS := $(shell $(PG_CONFIG) --pgxs) endif -# We need to do various things with the PostgreSQLl version. +# We need to do various things with the PostgreSQL version. VERSION = $(shell $(PG_CONFIG) --version | awk '{print $$2}') -# We support 8.1 and later. +# +# Major version check +# +# TODO: update this +# TODO9.1: update the $(TB_DIR) target below when de-supporting 9.1 ifeq ($(shell echo $(VERSION) | grep -qE "^(7[.]|8[.]0)" && echo yes || echo no),yes) $(error pgTAP requires PostgreSQL 8.1 or later. This is $(VERSION)) endif @@ -68,6 +118,53 @@ else include $(PGXS) endif +# +# DISABLED TESTS +# + +# Row security policy tests not supported by 9.4 and earlier. +ifeq ($(shell echo $(VERSION) | grep -qE "^9[.][01234]|8[.]" && echo yes || echo no),yes) +EXCLUDE_TEST_FILES += test/sql/policy.sql +endif + +# Partition tests tests not supported by 9.x and earlier. +ifeq ($(shell echo $(VERSION) | grep -qE "[89][.]" && echo yes || echo no),yes) +EXCLUDE_TEST_FILES += test/sql/partitions.sql +endif + +# Enum tests not supported by 8.2 and earlier. +ifeq ($(shell echo $(VERSION) | grep -qE "8[.][12]" && echo yes || echo no),yes) +EXCLUDE_TEST_FILES += test/sql/enumtap.sql +endif + +# Values tests not supported by 8.1 and earlier. +ifeq ($(shell echo $(VERSION) | grep -qE "8[.][1]" && echo yes || echo no),yes) +EXCLUDE_TEST_FILES += test/sql/valueset.sql +endif + +# +# Check for missing extensions +# +# NOTE! This currently MUST be after PGXS! The problem is that +# $(DESTDIR)$(datadir) aren't being expanded. +# +EXTENSION_DIR = $(DESTDIR)$(datadir)/extension +extension_control = $(shell file="$(EXTENSION_DIR)/$1.control"; [ -e "$$file" ] && echo "$$file") +ifeq (,$(call extension_control,citext)) +MISSING_EXTENSIONS += citext +endif +ifeq (,$(call extension_control,isn)) +MISSING_EXTENSIONS += isn +endif +ifeq (,$(call extension_control,ltree)) +MISSING_EXTENSIONS += ltree +endif +EXTENSION_TEST_FILES += test/sql/extension.sql +ifneq (,$(MISSING_EXTENSIONS)) +# NOTE: we emit a warning about this down below, but only when we're actually running a test. +EXCLUDE_TEST_FILES += $(EXTENSION_TEST_FILES) +endif + # We need Perl. ifneq (,$(findstring missing,$(PERL))) PERL := $(shell which perl) @@ -88,36 +185,23 @@ $(warning must be installed from CPAN. To do so, simply run:) $(warning cpan TAP::Parser::SourceHandler::pgTAP) endif -# Enum tests not supported by 8.2 and earlier. -ifeq ($(shell echo $(VERSION) | grep -qE "^8[.][12]" && echo yes || echo no),yes) -TESTS := $(filter-out test/sql/enumtap.sql,$(TESTS)) -REGRESS := $(filter-out enumtap,$(REGRESS)) -endif - -# Values tests not supported by 8.1 and earlier. -ifeq ($(shell echo $(VERSION) | grep -qE "^8[.][1]" && echo yes || echo no),yes) -TESTS := $(filter-out test/sql/enumtap.sql sql/valueset.sql,$(TESTS)) -REGRESS := $(filter-out enumtap valueset,$(REGRESS)) -endif - -# Partition tests tests not supported by 9.x and earlier. -ifeq ($(shell echo $(VERSION) | grep -qE "^[89][.]" && echo yes || echo no),yes) -TESTS := $(filter-out test/sql/partitions.sql,$(TESTS)) -REGRESS := $(filter-out partitions,$(REGRESS)) -endif - -# Row security policy tests not supported by 9.4 and earlier. -ifeq ($(shell echo $(VERSION) | grep -qE "^9[.][01234]|8[.]" && echo yes || echo no),yes) -TESTS := $(filter-out test/sql/policy.sql,$(TESTS)) -REGRESS := $(filter-out policy,$(REGRESS)) -endif +# Use a build directory to avoid cluttering up the main repo. (Maybe should just switch to VPATH builds?) +# WARNING! Not everything uses this! TODO: move all targets into $(B_DIR) +B_DIR ?= .build # Determine the OS. Borrowed from Perl's Configure. OSNAME := $(shell $(SHELL) ./getos.sh) .PHONY: test -sql/pgtap.sql: sql/pgtap.sql.in test/setup.sql +# TARGET uninstall-all: remove ALL installed versions of pgTap (rm pgtap*). +# Unlike `make unintall`, this removes pgtap*, not just our defined targets. +# Useful when testing multiple versions of pgtap. +uninstall-all: + rm -f $(EXTENSION_DIR)/pgtap* + +# TODO: switch this whole thing to a perl or shell script that understands the file naming convention and how to compare that to $VERSION. +sql/pgtap.sql: sql/pgtap.sql.in cp $< $@ ifeq ($(shell echo $(VERSION) | grep -qE "^(9[.][0123456]|8[.][1234])" && echo yes || echo no),yes) patch -p0 < compat/install-9.6.patch @@ -149,7 +233,7 @@ endif sed -e 's,MODULE_PATHNAME,$$libdir/pgtap,g' -e 's,__OS__,$(OSNAME),g' -e 's,__VERSION__,$(NUMVERSION),g' sql/pgtap.sql > sql/pgtap.tmp mv sql/pgtap.tmp sql/pgtap.sql -# Ugly hacks for now... +# Ugly hacks for now... TODO: script that understands $VERSION and will apply all the patch files for that version EXTRA_CLEAN += sql/pgtap--0.99.0--1.0.0.sql sql/pgtap--0.99.0--1.0.0.sql: sql/pgtap--0.99.0--1.0.0.sql.in cp $< $@ @@ -191,19 +275,20 @@ endif sql/uninstall_pgtap.sql: sql/pgtap.sql test/setup.sql grep '^CREATE ' sql/pgtap.sql | $(PERL) -e 'for (reverse ) { chomp; s/CREATE (OR REPLACE )?/DROP /; print "$$_;\n" }' | sed 's/DROP \(FUNCTION\|VIEW\|TYPE\) /DROP \1 IF EXISTS /' > sql/uninstall_pgtap.sql +# +# Support for static install files +# + +# The use of $@.tmp is to eliminate the possibility of leaving an invalid pgtap-static.sql in case the recipe fails part-way through. +# TODO: the sed command needs the equivalent of bash's PIPEFAIL; should just replace this with some perl magic sql/pgtap-static.sql: sql/pgtap.sql.in - cp $< $@ - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-10.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-9.6.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-9.4.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-9.2.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-9.1.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-9.0.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-8.4.patch | patch -p0 - sed -e 's,sql/pgtap,sql/pgtap-static,g' compat/install-8.3.patch | patch -p0 - sed -e 's,MODULE_PATHNAME,$$libdir/pgtap,g' -e 's,__OS__,$(OSNAME),g' -e 's,__VERSION__,$(NUMVERSION),g' $@ > sql/pgtap-static.tmp - mv sql/pgtap-static.tmp $@ -EXTRA_CLEAN += sql/pgtap-static.sql + cp $< $@.tmp + for p in `ls compat/install-*.patch | sort -rn`; do \ + echo; echo '***' "Patching pgtap-static.sql with $$p"; \ + sed -e 's#sql/pgtap.sql#sql/pgtap-static.sql.tmp#g' "$$p" | patch -p0; \ + done + sed -e 's#MODULE_PATHNAME#$$libdir/pgtap#g' -e 's#__OS__#$(OSNAME)#g' -e 's#__VERSION__#$(NUMVERSION)#g' $@.tmp > $@ +EXTRA_CLEAN += sql/pgtap-static.sql sql/pgtap-static.sql.tmp sql/pgtap-core.sql: sql/pgtap-static.sql $(PERL) compat/gencore 0 sql/pgtap-static.sql > sql/pgtap-core.sql @@ -211,18 +296,189 @@ sql/pgtap-core.sql: sql/pgtap-static.sql sql/pgtap-schema.sql: sql/pgtap-static.sql $(PERL) compat/gencore 1 sql/pgtap-static.sql > sql/pgtap-schema.sql +$(B_DIR)/static/: $(B_DIR) + mkdir -p $@ + +# We don't lump this in with the $(B_DIR)/static target because that would run the risk of a failure of the cp command leaving an empty directory behind +$(B_DIR)/static/%/: %/ $(B_DIR)/static + cp -R $< $@ + # Make sure that we build the regression tests. installcheck: test/setup.sql -# In addition to installcheck, one can also run the tests through pg_prove. -test: test/setup.sql - # Filter-out tests that intentionally fail. They should be tested by installcheck. - pg_prove --pset tuples_only=1 $(filter-out test/sql/run%,$(TESTS)) - html: multimarkdown doc/pgtap.mmd > doc/pgtap.html ./tocgen doc/pgtap.html 2> doc/toc.html perl -MPod::Simple::XHTML -E "my \$$p = Pod::Simple::XHTML->new; \$$p->html_header_tags(''); \$$p->strip_verbatim_indent(sub { my \$$l = shift; (my \$$i = \$$l->[0]) =~ s/\\S.*//; \$$i }); \$$p->parse_from_file('`perldoc -l pg_prove`')" > doc/pg_prove.html +# +# Actual test targets +# + +# TARGET regress: run installcheck then print any diffs from expected output. +.PHONY: regress +regress: installcheck_deps + $(MAKE) installcheck || ([ -e regression.diffs ] && $${PAGER:-cat} regression.diffs; exit 1) + +# TARGET updatecheck: install an older version of pgTap from PGXN (controlled by $UPDATE_FROM; 0.95.0 by default), update to the current version via ALTER EXTENSION, then run installcheck. +.PHONY: updatecheck +updatecheck: updatecheck_deps install + $(MAKE) updatecheck_run || ([ -e regression.diffs ] && $${PAGER:-cat} regression.diffs; exit 1) + +# General dependencies for installcheck. Note that several other places add themselves as dependencies. +.PHONY: installcheck_deps +installcheck_deps: $(SCHEDULE_DEST_FILES) extension_check set_parallel_conn # More dependencies below + +# In addition to installcheck, one can also run the tests through pg_prove. +test: extension_check + pg_prove --pset tuples_only=1 $(TEST_FILES) + +# +# General test support +# +# In order to support parallel testing, we need to have a test schedule file, +# which we build dynamically. Instead of cluttering the test directory, we use +# a test build directiory ($(TB_DIR)). We also use $(TB_DIR) to drop some +# additional artifacts, so that we can automatically determine if certain +# dependencies (such as excluded tests) have changed since the last time we +# ran. +TB_DIR = test/build +GENERATED_SCHEDULE_DEPS = $(TB_DIR)/tests $(TB_DIR)/exclude_tests +REGRESS = --schedule $(TB_DIR)/run.sch # Set this again just to be safe +REGRESS_OPTS = --inputdir=test --load-language=plpgsql --max-connections=$(PARALLEL_CONN) --schedule $(SETUP_SCH) $(REGRESS_CONF) +SETUP_SCH = test/schedule/main.sch # schedule to use for test setup; this can be forcibly changed by some targets! +IGNORE_TESTS = $(notdir $(EXCLUDE_TEST_FILES:.sql=)) +PARALLEL_TESTS = $(filter-out $(IGNORE_TESTS),$(filter-out $(SERIAL_TESTS),$(TESTS))) +GENERATED_SCHEDULES = $(TB_DIR)/serial.sch $(TB_DIR)/parallel.sch +installcheck: $(TB_DIR)/run.sch installcheck_deps + +# Parallel tests will use $(PARALLEL_TESTS) number of connections if we let it, +# but max_connections may not be set that high. You can set this manually to 1 +# for no parallelism +# +# This can be a bit expensive if we're not testing, so set it up as a +# dependency of installcheck +.PHONY: set_parallel_conn +set_parallel_conn: + $(eval PARALLEL_CONN = $(shell tools/parallel_conn.sh $(PARALLEL_CONN))) + @[ -n "$(PARALLEL_CONN)" ] + @echo "Using $(PARALLEL_CONN) parallel test connections" + +# Have to do this as a separate task to ensure the @[ -n ... ] test in set_parallel_conn actually runs +$(TB_DIR)/which_schedule: $(TB_DIR)/ set_parallel_conn + $(eval SCHEDULE = $(shell [ $(PARALLEL_CONN) -gt 1 ] && echo $(TB_DIR)/parallel.sch || echo $(TB_DIR)/serial.sch)) + @[ -n "$(SCHEDULE)" ] + @[ "`cat $@ 2>/dev/null`" = "$(SCHEDULE)" ] || (echo "Schedule changed to $(SCHEDULE)"; echo "$(SCHEDULE)" > $@) + +# Generated schedule files, one for serial one for parallel +.PHONY: $(TB_DIR)/tests # Need this target to force schedule rebuild if $(TEST) changes +$(TB_DIR)/tests: $(TB_DIR)/ + @[ "`cat $@ 2>/dev/null`" = "$(TEST)" ] || (echo "Rebuilding $@"; echo "$(TEST)" > $@) + +.PHONY: $(TB_DIR)/exclude_tests # Need this target to force schedule rebuild if $(EXCLUDE_TEST) changes +$(TB_DIR)/exclude_tests: $(TB_DIR)/ + @[ "`cat $@ 2>/dev/null`" = "$(EXCLUDE_TEST)" ] || (echo "Rebuilding $@"; echo "$(EXCLUDE_TEST)" > $@) + +$(TB_DIR)/serial.sch: $(GENERATED_SCHEDULE_DEPS) + @(for f in $(IGNORE_TESTS); do echo "ignore: $$f"; done; for f in $(TESTS); do echo "test: $$f"; done) > $@ + +$(TB_DIR)/parallel.sch: $(GENERATED_SCHEDULE_DEPS) + @( \ + for f in $(SERIAL_TESTS); do echo "test: $$f"; done; \ + ([ -z "$(IGNORE_TESTS)" ] || echo "ignore: $(IGNORE_TESTS)"); \ + ([ -z "$(PARALLEL_TESTS)" ] || echo "test: $(PARALLEL_TESTS)") \ + ) > $@ + +$(TB_DIR)/run.sch: $(TB_DIR)/which_schedule $(GENERATED_SCHEDULES) + cp `cat $<` $@ + +# Don't generate noise if we're not running tests... +.PHONY: extension_check +extension_check: + @[ -z "$(MISSING_EXTENSIONS)" ] || (echo; echo '***************************'; echo "WARNING: Some mandatory extensions ($(MISSING_EXTENSIONS)) are not installed; ignoring tests: $(EXTENSION_TEST_FILES)"; echo '***************************'; echo) + + +# These tests have specific dependencies +test/sql/build.sql: sql/pgtap.sql +test/sql/create.sql test/sql/update.sql: pgtap-version-$(EXTVERSION) + +test/sql/%.sql: test/schedule/%.sql + @(echo '\unset ECHO'; echo '-- GENERATED FILE! DO NOT EDIT!'; echo "-- Original file: $<"; cat $< ) > $@ + +# Prior to 9.2, EXTRA_CLEAN just does rm -f, which obviously won't work with a directory. +# TODO9.1: switch back to EXTRA_CLEAN when removing support for 9.1 +#EXTRA_CLEAN += $(TB_DIR)/ +clean: clean_tb_dir +.PHONY: clean_tb_dir +clean_tb_dir: + @rm -rf $(TB_DIR) +$(TB_DIR)/: + @mkdir -p $@ + +clean: clean_b_dir +.PHONY: clean_b_dir +clean_b_dir: + @rm -rf $(B_DIR) +$(B_DIR)/: + @mkdir -p $@ + + +# +# Update test support +# + +# If the specified version of pgtap doesn't exist, install it. Note that the +# real work is done by the $(EXTENSION_DIR)/pgtap--%.sql rule below. +pgtap-version-%: $(EXTENSION_DIR)/pgtap--%.sql + @true # Necessary to have a fake action here + + +# Travis will complain if we reinstall too quickly, so don't run make install +# unless actually necessary. +$(EXTENSION_DIR)/pgtap--$(EXTVERSION).sql: sql/pgtap--$(EXTVERSION).sql + $(MAKE) install + +# Need to explicitly exclude the current version. I wonder if there's a way to do this with % in the target? +# Note that we need to capture the test failure so the rule doesn't abort +$(EXTENSION_DIR)/pgtap--%.sql: + @ver=$(@:$(EXTENSION_DIR)/pgtap--%.sql=%); [ "$$ver" = "$(EXTVERSION)" ] || (echo Installing pgtap version $$ver from pgxn; pgxn install pgtap=$$ver) + +# This is separated out so it can be called before calling updatecheck_run +.PHONY: updatecheck_deps +updatecheck_deps: pgtap-version-$(UPDATE_FROM) test/sql/update.sql + +# We do this as a separate step to change SETUP_SCH before the main updatecheck +# recipe calls installcheck (which depends on SETUP_SCH being set correctly). +.PHONY: updatecheck_setup +# pg_regress --launcher not supported prior to 9.1 +# There are some other failures in 9.1 and 9.2 (see https://travis-ci.org/decibel/pgtap/builds/358206497). +# TODO: find something that can generically compare majors (ie: GE91 from +# https://github.com/decibel/pgxntool/blob/master/base.mk). +updatecheck_setup: updatecheck_deps + @if echo $(VERSION) | grep -qE "8[.]|9[.][012]"; then echo "updatecheck is not supported prior to 9.3"; exit 1; fi + $(eval SETUP_SCH = test/schedule/update.sch) + $(eval REGRESS_OPTS += --launcher "tools/psql_args.sh -v 'old_ver=$(UPDATE_FROM)' -v 'new_ver=$(EXTVERSION)'") + @echo + @echo "###################" + @echo "Testing upgrade from $(UPDATE_FROM) to $(EXTVERSION)" + @echo "###################" + @echo + +.PHONY: updatecheck_run +updatecheck_run: updatecheck_setup installcheck + +# +# STOLEN FROM pgxntool +# + +# TARGET results: runs `make test` and copies all result files to test/expected/. DO NOT RUN THIS UNLESS YOU'RE CERTAIN ALL YOUR TESTS ARE PASSING! +.PHONY: results +results: installcheck result-rsync + +.PHONY: +result-rsync: + rsync -rlpgovP results/ test/expected + + # To use this, do make print-VARIABLE_NAME print-% : ; $(info $* is $(flavor $*) variable set to "$($*)") @true diff --git a/README.md b/README.md index 9839c83c5437..f6db0622f0ba 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ documentation in `doc/pgtap.mmd` or To build it, just do this: make - make installcheck make install + make installcheck If you encounter an error such as: diff --git a/pg-travis-test.sh b/pg-travis-test.sh new file mode 100644 index 000000000000..c9784920d7da --- /dev/null +++ b/pg-travis-test.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Based on https://gist.github.com/petere/6023944 + +set -eux +failed='' + +sudo apt-get update + +packages="python-setuptools postgresql-$PGVERSION postgresql-server-dev-$PGVERSION postgresql-common" + +# bug: http://www.postgresql.org/message-id/20130508192711.GA9243@msgid.df7cb.de +sudo update-alternatives --remove-all postmaster.1.gz + +# stop all existing instances (because of https://github.com/travis-ci/travis-cookbooks/pull/221) +sudo service postgresql stop +# and make sure they don't come back +echo 'exit 0' | sudo tee /etc/init.d/postgresql +sudo chmod a+x /etc/init.d/postgresql + +sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install $packages + +export PGPORT=55435 +export PGUSER=postgres +export PG_CONFIG=/usr/lib/postgresql/$PGVERSION/bin/pg_config +sudo pg_createcluster --start $PGVERSION test -p $PGPORT -- -A trust + +sudo easy_install pgxnclient + +test_make() { + set +ux + # Many tests depend on install, so just use sudo for all of them + if ! sudo make "$@"; then + echo + echo '!!!!!!!!!!!!!!!!' + echo "make $@ failed" + echo '!!!!!!!!!!!!!!!!' + echo + failed="$failed '$@'" + fi + set -ux +} + +test_make clean regress + +# pg_regress --launcher not supported prior to 9.1 +# There are some other failures in 9.1 and 9.2 (see https://travis-ci.org/decibel/pgtap/builds/358206497). +echo $PGVERSION | grep -qE "8[.]|9[.][012]" || test_make clean updatecheck + +# Explicitly test these other targets + +# TODO: install software necessary to allow testing the 'test' and 'html' targets +for t in all install ; do + test_make clean $t + test_make $t +done + +if [ -n "$failed" ]; then + set +ux + # $failed will have a leading space if it's not empty + echo "These test targets failed:$failed" + exit 1 +fi diff --git a/sql/pgtap--0.97.0--0.98.0.sql.in b/sql/pgtap--0.97.0--0.98.0.sql.in index a921eaf62271..d0a7b53f82d0 100644 --- a/sql/pgtap--0.97.0--0.98.0.sql.in +++ b/sql/pgtap--0.97.0--0.98.0.sql.in @@ -233,6 +233,23 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- table_owner_is ( table, user, description ) +CREATE OR REPLACE FUNCTION table_owner_is ( NAME, NAME, TEXT ) +RETURNS TEXT AS $$ +DECLARE + owner NAME := _get_rel_owner('{r,p}'::char[], $1); +BEGIN + -- Make sure the table exists. + IF owner IS NULL THEN + RETURN ok(FALSE, $3) || E'\n' || diag( + E' Table ' || quote_ident($1) || ' does not exist' + ); + END IF; + + RETURN is(owner, $2, $3); +END; +$$ LANGUAGE plpgsql; + -- is_partitioned( schema, table, description ) CREATE OR REPLACE FUNCTION is_partitioned ( NAME, NAME, TEXT ) RETURNS TEXT AS $$ diff --git a/sql/pgtap--0.98.0--0.99.0.sql.in b/sql/pgtap--0.98.0--0.99.0.sql.in index e165e17bbad0..1c957ac329ff 100644 --- a/sql/pgtap--0.98.0--0.99.0.sql.in +++ b/sql/pgtap--0.98.0--0.99.0.sql.in @@ -83,6 +83,26 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Note: this fixes a bug in the 97->98 upgrade script +-- table_owner_is ( table, user, description ) +/* +CREATE OR REPLACE FUNCTION table_owner_is ( NAME, NAME, TEXT ) +RETURNS TEXT AS $$ +DECLARE + owner NAME := _get_rel_owner('{r,p}'::char[], $1); +BEGIN + -- Make sure the table exists. + IF owner IS NULL THEN + RETURN ok(FALSE, $3) || E'\n' || diag( + E' Table ' || quote_ident($1) || ' does not exist' + ); + END IF; + + RETURN is(owner, $2, $3); +END; +$$ LANGUAGE plpgsql; +*/ + -- _hasc( schema, table, constraint_type ) CREATE OR REPLACE FUNCTION _hasc ( NAME, NAME, CHAR ) RETURNS BOOLEAN AS $$ diff --git a/test/expected/build.out b/test/expected/build.out new file mode 100644 index 000000000000..f9861fe7708c --- /dev/null +++ b/test/expected/build.out @@ -0,0 +1 @@ +\unset ECHO diff --git a/test/expected/create.out b/test/expected/create.out new file mode 100644 index 000000000000..f9861fe7708c --- /dev/null +++ b/test/expected/create.out @@ -0,0 +1 @@ +\unset ECHO diff --git a/test/expected/extension.out b/test/expected/extension.out index c31ecf60467e..80d6fa05d589 100644 --- a/test/expected/extension.out +++ b/test/expected/extension.out @@ -12,9 +12,9 @@ ok 9 - extensions_are(exts, desc) should have the proper diagnostics ok 10 - extensions_are(exts) should pass ok 11 - extensions_are(exts) should have the proper description ok 12 - extensions_are(exts) should have the proper diagnostics -ok 13 - extensions_are(public, exts, desc) should pass -ok 14 - extensions_are(public, exts, desc) should have the proper description -ok 15 - extensions_are(public, exts, desc) should have the proper diagnostics +ok 13 - extensions_are(ci_schema, exts, desc) should pass +ok 14 - extensions_are(ci_schema, exts, desc) should have the proper description +ok 15 - extensions_are(ci_schema, exts, desc) should have the proper diagnostics ok 16 - extensions_are(non-sch, exts) should pass ok 17 - extensions_are(non-sch, exts) should have the proper description ok 18 - extensions_are(non-sch, exts) should have the proper diagnostics diff --git a/test/expected/runjusttests_6.out b/test/expected/runjusttests_6.out new file mode 100644 index 000000000000..5dac15ac91b3 --- /dev/null +++ b/test/expected/runjusttests_6.out @@ -0,0 +1,41 @@ +\unset ECHO + # Subtest: whatever."test ident"() + ok 1 - ident + ok 2 - ident 2 + 1..2 +ok 1 - whatever."test ident" + # Subtest: whatever.testnada() + 1..0 + # No tests run! +not ok 2 - whatever.testnada +# Failed test 2: "whatever.testnada" + # Subtest: whatever.testplpgsql() + ok 1 - plpgsql simple + ok 2 - plpgsql simple 2 + ok 3 - Should be a 1 in the test table + 1..3 +ok 3 - whatever.testplpgsql + # Subtest: whatever.testplpgsqldie() + # Test died: This test should die, but not halt execution. +# Note that in some cases we get what appears to be a duplicate context message, but that is due to Postgres itself. +not ok 4 - whatever.testplpgsqldie +# Failed test 4: "whatever.testplpgsqldie" + # Subtest: whatever.testthis() + ok 1 - simple pass + ok 2 - another simple pass + 1..2 +ok 5 - whatever.testthis + # Subtest: whatever.testy() + ok 1 - pass + not ok 2 - this test intentionally fails + # Failed test 2: "this test intentionally fails" + 1..2 + # Looks like you failed 1 tests of 2 +not ok 6 - whatever.testy +# Failed test 6: "whatever.testy" + # Subtest: whatever.testz() + ok 1 - Late test should find nothing in the test table + 1..1 +ok 7 - whatever.testz +1..7 +# Looks like you failed 3 tests of 7 diff --git a/test/expected/update.out b/test/expected/update.out new file mode 100644 index 000000000000..4bd0c77c6d80 --- /dev/null +++ b/test/expected/update.out @@ -0,0 +1,4 @@ +\unset ECHO +1..2 +ok 1 - Old version of pgtap correctly installed +ok 2 - pgtap correctly updated diff --git a/test/psql.sql b/test/psql.sql new file mode 100644 index 000000000000..c147815d44f7 --- /dev/null +++ b/test/psql.sql @@ -0,0 +1,14 @@ +\set QUIET 1 + +-- +-- Tests for pgTAP. +-- +-- +-- Format the output for nice TAP. +\pset format unaligned +\pset tuples_only true +\pset pager + +-- Revert all changes on failure. +\set ON_ERROR_ROLLBACK 1 +\set ON_ERROR_STOP true diff --git a/test/schedule/build.sql b/test/schedule/build.sql new file mode 100644 index 000000000000..9365ba833bda --- /dev/null +++ b/test/schedule/build.sql @@ -0,0 +1,10 @@ +\unset ECHO +\i test/psql.sql + +/* + * Presumably no one is installing this way anymore, but this is a nice way to + * pick up any syntax errors during install. + */ +BEGIN; +\i sql/pgtap.sql +ROLLBACK; diff --git a/test/schedule/create.sql b/test/schedule/create.sql new file mode 100644 index 000000000000..ba355ed24dab --- /dev/null +++ b/test/schedule/create.sql @@ -0,0 +1,3 @@ +\unset ECHO +\i test/psql.sql +CREATE EXTENSION pgtap; diff --git a/test/schedule/main.sch b/test/schedule/main.sch new file mode 100644 index 000000000000..a8a5fbcaf964 --- /dev/null +++ b/test/schedule/main.sch @@ -0,0 +1,2 @@ +test: build +test: create diff --git a/test/schedule/update.sch b/test/schedule/update.sch new file mode 100644 index 000000000000..ea566141772f --- /dev/null +++ b/test/schedule/update.sch @@ -0,0 +1 @@ +test: update diff --git a/test/schedule/update.sql b/test/schedule/update.sql new file mode 100644 index 000000000000..be73d5174abf --- /dev/null +++ b/test/schedule/update.sql @@ -0,0 +1,20 @@ +\unset ECHO +\i test/psql.sql + +CREATE EXTENSION pgtap VERSION :'old_ver'; +SELECT plan(2); +SELECT is( + (SELECT extversion FROM pg_extension WHERE extname = 'pgtap') + , :'old_ver' + , 'Old version of pgtap correctly installed' +); + +ALTER EXTENSION pgtap UPDATE TO :'new_ver'; + +SELECT is( + (SELECT extversion FROM pg_extension WHERE extname = 'pgtap') + , :'new_ver' + , 'pgtap correctly updated' +); + +SELECT finish(); diff --git a/test/setup.sql b/test/setup.sql index 6cbbc964a5cc..492e91d83b5a 100644 --- a/test/setup.sql +++ b/test/setup.sql @@ -1,23 +1,3 @@ -\set QUIET 1 - --- --- Tests for pgTAP. --- --- --- Format the output for nice TAP. -\pset format unaligned -\pset tuples_only true -\pset pager - --- Revert all changes on failure. -\set ON_ERROR_ROLLBACK 1 -\set ON_ERROR_STOP true +\i test/psql.sql BEGIN; - --- Uncomment when testing with PGOPTIONS=--search_path=tap --- CREATE SCHEMA tap; SET search_path TO tap,public; - --- Load the TAP functions. -\i sql/pgtap.sql ---CREATE EXTENSION pgtap; diff --git a/test/sql/.gitignore b/test/sql/.gitignore new file mode 100644 index 000000000000..f7529a75b71a --- /dev/null +++ b/test/sql/.gitignore @@ -0,0 +1,3 @@ +build.sql +create.sql +update.sql diff --git a/test/sql/extension.sql b/test/sql/extension.sql index 0269a3dbca94..36f8ee94f7e5 100644 --- a/test/sql/extension.sql +++ b/test/sql/extension.sql @@ -15,8 +15,9 @@ BEGIN IF pg_version_num() >= 90100 THEN EXECUTE $E$ CREATE SCHEMA someschema; + CREATE SCHEMA ci_schema; CREATE SCHEMA "empty schema"; - CREATE EXTENSION IF NOT EXISTS citext; + CREATE EXTENSION IF NOT EXISTS citext SCHEMA ci_schema; CREATE EXTENSION IF NOT EXISTS isn SCHEMA someschema; CREATE EXTENSION IF NOT EXISTS ltree SCHEMA someschema; $E$; @@ -38,7 +39,7 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT* FROM check_test( - extensions_are( ARRAY['citext', 'isn', 'ltree', 'plpgsql'], 'Got em' ), + extensions_are( ARRAY['citext', 'isn', 'ltree', 'plpgsql', 'pgtap'], 'Got em' ), true, 'extensions_are(exts, desc)', 'Got em', @@ -46,7 +47,7 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT* FROM check_test( - extensions_are( ARRAY['citext', 'isn', 'ltree', 'plpgsql'] ), + extensions_are( ARRAY['citext', 'isn', 'ltree', 'plpgsql', 'pgtap'] ), true, 'extensions_are(exts)', 'Should have the correct extensions', @@ -54,9 +55,9 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT* FROM check_test( - extensions_are( 'public', ARRAY['citext'], 'Got em' ), + extensions_are( 'ci_schema', ARRAY['citext'], 'Got em' ), true, - 'extensions_are(public, exts, desc)', + 'extensions_are(ci_schema, exts, desc)', 'Got em', '' ) AS b LOOP RETURN NEXT tap.b; END LOOP; @@ -83,7 +84,7 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT* FROM check_test( - extensions_are( ARRAY['citext', 'isn', 'ltree', 'nonesuch'] ), + extensions_are( ARRAY['citext', 'isn', 'ltree', 'pgtap', 'nonesuch'] ), false, 'extensions_are(someexts)', 'Should have the correct extensions', @@ -98,7 +99,7 @@ BEGIN -- 8 tests FOR tap IN SELECT * FROM check_test( - has_extension( 'public', 'citext', 'desc' ), + has_extension( 'ci_schema', 'citext', 'desc' ), true, 'has_extension( schema, name, desc )', 'desc', @@ -106,10 +107,10 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT * FROM check_test( - has_extension( 'public', 'citext'::name ), + has_extension( 'ci_schema', 'citext'::name ), true, 'has_extension( schema, name )', - 'Extension citext should exist in schema public', + 'Extension citext should exist in schema ci_schema', '' ) AS b LOOP RETURN NEXT tap.b; END LOOP; @@ -198,7 +199,7 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT * FROM check_test( - hasnt_extension( 'public', 'citext', 'desc' ), + hasnt_extension( 'ci_schema', 'citext', 'desc' ), false, 'hasnt_extension( schema, name, desc )', 'desc', @@ -206,10 +207,10 @@ BEGIN ) AS b LOOP RETURN NEXT tap.b; END LOOP; FOR tap IN SELECT * FROM check_test( - hasnt_extension( 'public', 'citext'::name ), + hasnt_extension( 'ci_schema', 'citext'::name ), false, 'hasnt_extension( schema, name )', - 'Extension citext should not exist in schema public', + 'Extension citext should not exist in schema ci_schema', '' ) AS b LOOP RETURN NEXT tap.b; END LOOP; @@ -264,7 +265,7 @@ BEGIN FOR tap IN SELECT * FROM check_test( pass('mumble'), true, - 'extensions_are(public, exts, desc)', + 'extensions_are(ci_schema, exts, desc)', 'mumble', '' ) AS b LOOP RETURN NEXT tap.b; END LOOP; diff --git a/test/sql/performs_ok.sql b/test/sql/performs_ok.sql index 5f9c8df5ae60..a62b101e79f2 100644 --- a/test/sql/performs_ok.sql +++ b/test/sql/performs_ok.sql @@ -23,10 +23,10 @@ SELECT * FROM check_test( ); SELECT * FROM check_test( - performs_ok( 'SELECT TRUE', 99.99 ), + performs_ok( 'SELECT TRUE', 199.99 ), true, 'simple select numeric', - 'Should run in less than 99.99 ms', + 'Should run in less than 199.99 ms', '' ); diff --git a/tools/parallel_conn.sh b/tools/parallel_conn.sh new file mode 100755 index 000000000000..5cad67e8df10 --- /dev/null +++ b/tools/parallel_conn.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +# Find the maximum safe connections for parallel execution + +error () { + echo $@ 1>&2 +} +die () { + error $@ + exit 1 +} + +[ $# -le 1 ] || die "$0: Invalid number of arguments" + +PARALLEL_CONN=$1 + +if [ -n "$PARALLEL_CONN" ]; then + [ $PARALLEL_CONN -ge 1 ] 2>/dev/null || die "Invalid value for PARALLEL_CONN ($PARALLEL_CONN)" + echo $PARALLEL_CONN + exit +fi + +COMMAND="SELECT greatest(1, current_setting('max_connections')::int - current_setting('superuser_reserved_connections')::int - (SELECT count(*) FROM pg_stat_activity) - 2)" + +if PARALLEL_CONN=`psql -d ${PGDATABASE:-postgres} -qtc "$COMMAND" 2> /dev/null`; then + if [ $PARALLEL_CONN -ge 1 ] 2>/dev/null; then + # We know it's a number at this point + [ $PARALLEL_CONN -eq 1 ] && error "NOTICE: unable to run tests in parallel; not enough connections" + echo $PARALLEL_CONN + exit + fi +fi + +error "Problems encountered determining maximum parallel test connections; forcing serial mode" +echo 1 diff --git a/tools/psql_args.sh b/tools/psql_args.sh new file mode 100755 index 000000000000..e1f203f0984e --- /dev/null +++ b/tools/psql_args.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# This script allows for passing args to psql using pg_regress's --launcher option + +while [ $# -gt 0 ]; do + case $1 in + */psql) + found=1 + break + ;; + *) + args="$args $1" + shift + ;; + esac +done + +if [ -z "$found" ]; then + echo "Error: psql not found in arguments" + exit 1 +fi + +#$* $args <&0 || exit $? +$* $args || exit $?