diff --git a/.drone.star b/.drone.star index 59c2fc14..76959b35 100644 --- a/.drone.star +++ b/.drone.star @@ -15,24 +15,15 @@ config = { "master", ], "appInstallCommand": "make", - "codestyle": { - "ordinary": { - "phpVersions": [ - "7.2", - "7.3", - "7.4", - ], - }, - }, + "codestyle": True, "phpunit": { "allDatabases": { "phpVersions": [ - "7.2", + "7.3", ], }, "reducedDatabases": { "phpVersions": [ - "7.3", "7.4", ], "databases": [ @@ -74,7 +65,7 @@ config = { "extraSetup": [ { "name": "configure-app", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd /var/www/owncloud/server", @@ -114,7 +105,7 @@ config = { "extraSetup": [ { "name": "configure-app", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd /var/www/owncloud/server", @@ -148,7 +139,7 @@ config = { "extraSetup": [ { "name": "configure-app", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd /var/www/owncloud/server", @@ -182,7 +173,7 @@ config = { "extraSetup": [ { "name": "configure-app", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd /var/www/owncloud/server", @@ -211,9 +202,6 @@ config = { "servers": [ "daily-master-qa", ], - "phpVersions": [ - "7.2", - ], "runCoreTests": True, "federatedServerNeeded": True, "cron": "nightly", @@ -231,9 +219,6 @@ config = { "servers": [ "daily-master-qa", ], - "phpVersions": [ - "7.2", - ], "runCoreTests": True, "cron": "nightly", "runAllSuites": True, @@ -251,9 +236,6 @@ config = { "servers": [ "daily-master-qa", ], - "phpVersions": [ - "7.2", - ], "emailNeeded": True, "runCoreTests": True, "federatedServerNeeded": True, @@ -268,7 +250,7 @@ config = { "extraServices": [ { "name": "dummy-clamav", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ # dummy clamav required for unit tests @@ -287,7 +269,7 @@ config = { "extraSetup": [ { "name": "wait-for-service", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "wait-for-it -t 600 dummy-clamav:5555", @@ -305,7 +287,7 @@ config = { "extraSetup": [ { "name": "configure-app", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd /var/www/owncloud/server", @@ -363,7 +345,7 @@ def main(ctx): return before + coverageTests + afterCoverageTests + nonCoverageTests + stages + after def beforePipelines(ctx): - return codestyle(ctx) + jscodestyle(ctx) + phpstan(ctx) + phan(ctx) + checkStarlark() + return codestyle(ctx) + jscodestyle(ctx) + checkForRecentBuilds(ctx) + phpstan(ctx) + phan(ctx) + phplint(ctx) + checkStarlark() def coveragePipelines(ctx): # All unit test pipelines that have coverage or other test analysis reported @@ -410,7 +392,7 @@ def codestyle(ctx): return pipelines default = { - "phpVersions": ["7.2"], + "phpVersions": ["7.3"], } if "defaults" in config: @@ -517,6 +499,57 @@ def jscodestyle(ctx): return pipelines +def checkForRecentBuilds(ctx): + pipelines = [] + + result = { + "kind": "pipeline", + "type": "docker", + "name": "stop-recent-builds", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": stopRecentBuilds(ctx), + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/tags/**", + "refs/pull/**", + ], + }, + } + + pipelines.append(result) + + return pipelines + +def stopRecentBuilds(ctx): + app = "%s/apps/%s" % (dir["server"], ctx.repo.name) + return [{ + "name": "stop-recent-builds", + "image": "drone/cli:alpine", + "pull": "always", + "environment": { + "DRONE_SERVER": "https://drone.owncloud.com", + "DRONE_TOKEN": { + "from_secret": "drone_token", + }, + "OWNCLOUD_APP": ctx.repo.slug, + }, + "commands": [ + "drone build ls %s --status running > %s/recentBuilds.txt" % (ctx.repo.slug, app), + "drone build info %s ${DRONE_BUILD_NUMBER} > %s/thisBuildInfo.txt" % (ctx.repo.slug, app), + "cd %s && ./.drone/cancelBuilds.sh" % app, + ], + "when": { + "event": [ + "pull_request", + ], + }, + }] + def phpstan(ctx): pipelines = [] @@ -524,9 +557,10 @@ def phpstan(ctx): return pipelines default = { - "phpVersions": ["7.2"], + "phpVersions": ["7.3"], "logLevel": "2", "extraApps": {}, + "enableApp": True, } if "defaults" in config: @@ -566,7 +600,7 @@ def phpstan(ctx): "steps": installCore(ctx, "daily-master-qa", "sqlite", False) + installApp(ctx, phpVersion) + installExtraApps(phpVersion, params["extraApps"]) + - setupServerAndApp(ctx, phpVersion, params["logLevel"]) + + setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + [ { "name": "phpstan", @@ -600,7 +634,7 @@ def phan(ctx): return pipelines default = { - "phpVersions": ["7.2", "7.3", "7.4"], + "phpVersions": ["7.3", "7.4"], } if "defaults" in config: @@ -671,7 +705,7 @@ def build(ctx): return pipelines default = { - "phpVersions": ["7.2"], + "phpVersions": ["7.3"], "commands": [ "make dist", ], @@ -768,6 +802,7 @@ def javascript(ctx, withCoverage): "extraCommandsBeforeTestRun": [], "extraTeardown": [], "skip": False, + "enableApp": True, } if "defaults" in config: @@ -809,12 +844,12 @@ def javascript(ctx, withCoverage): }, "steps": installCore(ctx, "daily-master-qa", "sqlite", False) + installApp(ctx, "7.4") + - setupServerAndApp(ctx, "7.4", params["logLevel"]) + + setupServerAndApp(ctx, "7.4", params["logLevel"], False, params["enableApp"]) + params["extraSetup"] + [ { "name": "js-tests", - "image": "owncloudci/php:8.0", + "image": "owncloudci/nodejs:%s" % getNodeJsVersion(), "pull": "always", "environment": params["extraEnvironment"], "commands": params["extraCommandsBeforeTestRun"] + [ @@ -869,7 +904,7 @@ def phpTests(ctx, testType, withCoverage): errorFound = False default = { - "phpVersions": ["7.2", "7.3", "7.4"], + "phpVersions": ["7.3", "7.4"], "databases": [ "sqlite", "mariadb:10.2", @@ -889,6 +924,7 @@ def phpTests(ctx, testType, withCoverage): "extraApps": {}, "extraTeardown": [], "skip": False, + "enableApp": True, } if "defaults" in config: @@ -981,7 +1017,7 @@ def phpTests(ctx, testType, withCoverage): "steps": installCore(ctx, "daily-master-qa", db, False) + installApp(ctx, phpVersion) + installExtraApps(phpVersion, params["extraApps"]) + - setupServerAndApp(ctx, phpVersion, params["logLevel"]) + + setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + setupCeph(params["cephS3"]) + setupScality(params["scalityS3"]) + params["extraSetup"] + @@ -1065,7 +1101,7 @@ def acceptance(ctx): default = { "servers": ["daily-master-qa", "latest"], "browsers": ["chrome"], - "phpVersions": ["7.2"], + "phpVersions": ["7.4"], "databases": ["mariadb:10.2"], "esVersions": ["none"], "federatedServerNeeded": False, @@ -1095,6 +1131,7 @@ def acceptance(ctx): "debugSuites": [], "skipExceptParts": [], "earlyFail": True, + "enableApp": True, } if "defaults" in config: @@ -1145,7 +1182,7 @@ def acceptance(ctx): params["extraSetup"] = [ { "name": "configure-app", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd /var/www/owncloud/server/apps/files_primary_s3", @@ -1311,7 +1348,7 @@ def acceptance(ctx): (installFederated(testConfig["server"], testConfig["phpVersion"], testConfig["logLevel"], testConfig["database"], federationDbSuffix) + owncloudLog("federated") if testConfig["federatedServerNeeded"] else []) + installApp(ctx, testConfig["phpVersion"]) + installExtraApps(testConfig["phpVersion"], testConfig["extraApps"]) + - setupServerAndApp(ctx, testConfig["phpVersion"], testConfig["logLevel"], testConfig["federatedServerNeeded"]) + + setupServerAndApp(ctx, testConfig["phpVersion"], testConfig["logLevel"], testConfig["federatedServerNeeded"], params["enableApp"]) + owncloudLog("server") + setupCeph(testConfig["cephS3"]) + setupScality(testConfig["scalityS3"]) + @@ -1608,10 +1645,11 @@ def elasticSearchService(esVersion): return [{ "name": "elasticsearch", - "image": "webhippie/elasticsearch:%s" % esVersion, + "image": "owncloudops/elasticsearch:%s" % esVersion, "pull": "always", "environment": { - "ELASTICSEARCH_PLUGINS_INSTALL": "ingest-attachment", + "ELASTICSEARCH_ROOT_LOG_LEVEL": "warn", + "ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK": "false", }, }] @@ -1724,6 +1762,13 @@ def getDbDatabase(db): return "owncloud" +def getNodeJsVersion(): + if "nodeJsVersion" not in config: + # We use nodejs 14 as the default + return "14" + else: + return config["nodeJsVersion"] + def cacheRestore(): return [{ "name": "cache-restore", @@ -1840,7 +1885,23 @@ def installApp(ctx, phpVersion): if "appInstallCommand" not in config: return [] - return [{ + if "buildJsDeps" not in config: + installJsDeps = False + else: + installJsDeps = config["buildJsDeps"] + + return [ + { + "name": "install-app-js-%s" % config["app"], + "image": "owncloudci/nodejs:%s" % getNodeJsVersion(), + "pull": "always", + "commands": [ + "cd /var/www/owncloud/server/apps/%s" % config["app"], + "make install-js-deps", + "make build-dev", + ], + }, + ] if installJsDeps else [] + [{ "name": "install-app-%s" % ctx.repo.name, "image": "owncloudci/php:%s" % phpVersion, "pull": "always", @@ -1850,7 +1911,7 @@ def installApp(ctx, phpVersion): ], }] -def setupServerAndApp(ctx, phpVersion, logLevel, federatedServerNeeded = False): +def setupServerAndApp(ctx, phpVersion, logLevel, federatedServerNeeded = False, enableApp = True): return [{ "name": "setup-server-%s" % ctx.repo.name, "image": "owncloudci/php:%s" % phpVersion, @@ -1858,7 +1919,7 @@ def setupServerAndApp(ctx, phpVersion, logLevel, federatedServerNeeded = False): "commands": [ "cd %s" % dir["server"], "php occ a:l", - "php occ a:e %s" % ctx.repo.name, + "php occ a:e %s" % ctx.repo.name if enableApp else "", "php occ a:e testing", "php occ a:l", "php occ config:system:set trusted_domains 1 --value=server", @@ -1885,7 +1946,7 @@ def setupCeph(serviceParams): return [{ "name": "setup-ceph", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": setupCommands + ([ "./apps/files_primary_s3/tests/drone/create-bucket.sh", @@ -1913,7 +1974,7 @@ def setupScality(serviceParams): return [{ "name": "setup-scality", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": setupCommands + ([ "php occ s3:create-bucket owncloud --accept-warning", @@ -1928,7 +1989,7 @@ def setupElasticSearch(esVersion): return [{ "name": "setup-es", - "image": "owncloudci/php:7.2", + "image": "owncloudci/php:7.4", "pull": "always", "commands": [ "cd %s" % dir["server"], @@ -2167,3 +2228,60 @@ def checkStarlark(): ], }, }] + +def phplint(ctx): + pipelines = [] + + if "phplint" not in config: + return pipelines + + if type(config["phplint"]) == "bool": + if not config["phplint"]: + return pipelines + + result = { + "kind": "pipeline", + "type": "docker", + "name": "lint-test", + "workspace": { + "base": "/var/www/owncloud", + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": installNPM() + + lintTest(), + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/tags/**", + "refs/pull/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def installNPM(): + return [{ + "name": "npm-install", + "image": "owncloudci/nodejs:%s" % getNodeJsVersion(), + "pull": "always", + "commands": [ + "yarn install --frozen-lockfile", + ], + }] + +def lintTest(): + return [{ + "name": "lint-test", + "image": "owncloudci/php:7.4", + "pull": "always", + "commands": [ + "make test-lint", + ], + }] diff --git a/.drone/cancelBuilds.sh b/.drone/cancelBuilds.sh new file mode 100755 index 00000000..2d06b8e2 --- /dev/null +++ b/.drone/cancelBuilds.sh @@ -0,0 +1,62 @@ +SCRIPT_PATH=$(dirname "$0") +SCRIPT_PATH=$( cd "${SCRIPT_PATH}" && pwd ) # normalized and made absolute + +if [ -z "${OWNCLOUD_APP}" ] +then + echo "OWNCLOUD_APP environment variable is required to be defined as the organisation and repo name in the format: owncloud/app_name" + exit 1 +fi +# The recentBuilds.txt file contains all the information of the PR whose builds are recently running. +# Therefore, to filter out the BuildID and the Reference from all the recent builds which are running, +# the following awk command is used and the output containing the BuildId and the Reference of each +# builds/prs are stored in the filteredDescriptions.txt file. +awk "/Build #|Ref: refs\/pull/" ${SCRIPT_PATH}/../recentBuilds.txt > ${SCRIPT_PATH}/filteredDescriptions.txt + +# The thisBuildInfo.txt file contains the information of the current Build, including the BuildID, Reference, and +# other information of that particular build. To get the reference number/ pr number of the recent build, +# the following awk command and grep commands are used, where the Reference part ("Ref: refs/pull/5496/head") +# is stored in the "thisBuildFiltered.txt" file. From the reference, only the pr/reference number is extracted into +# the "NUMBER" variable. +awk "/Ref: refs\/pull/" ${SCRIPT_PATH}/../thisBuildInfo.txt > ${SCRIPT_PATH}/thisBuildFiltered.txt +NUMBER=$(grep -o -E "[0-9]+" ${SCRIPT_PATH}/thisBuildFiltered.txt) +referenceNum="Ref: refs/pull/"$NUMBER"/head" + +# From all the recent builds, the information about the BuildID and the reference which was stored in the +# "filteredDescriptions.txt" file, only the BuildID part from each of the buildInformation whose Reference is +# exactly same as that of the current build reference is filtered and stored in the "buildsToStop.txt" file. +# The buildIDs and the reference information of each build is stored sequentially in the "filteredDescriptions.txt" file as: + +# "Build #" +# "" +# "Build #" +# "" +# "Build #" +# "" + +# Therefore the following awk command extracts each lines just above the expected reference information. +awk -v ref="$referenceNum" 'index($0,ref){print p} {p=$0}' ${SCRIPT_PATH}/filteredDescriptions.txt > ${SCRIPT_PATH}/buildsToStop.txt + + +# The "buildsToStop.txt" file now contains the buildIDs of the recent builds whose reference id was equal to the reference of the +# current pr/build. + +# "Build #" +# "Build #" +# "Build #" +# "Build #" +# "Build #" + +# For each build in the "buildsToStop.txt" file, if the build number is older than the current drone build number, the build is cancelled. + +while IFS="" read -r p || [ -n "$p" ] +do + printf '%s\n' "$p" + buildNumber=$(echo "$p" | awk -F'#' '{print $(2)}') + + if [ $DRONE_BUILD_NUMBER \> "$buildNumber" ] + then + echo "CANCELLING BUILD: " $buildNumber + drone build stop $OWNCLOUD_APP $buildNumber + fi + +done <${SCRIPT_PATH}/buildsToStop.txt diff --git a/Makefile b/Makefile index 5c039256..59ed3920 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,6 @@ # Makefile for building the project COMPOSER_BIN := $(shell command -v composer 2> /dev/null) -ifndef COMPOSER_BIN - $(error composer is not available on your system, please install composer) -endif app_name=files_antivirus build_dir=$(CURDIR)/build diff --git a/composer.json b/composer.json index a7dd1aaa..e809cefe 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "description": "files_antivirus is an antivirus app for ownCloud based on ClamAV", "config": { "platform": { - "php": "7.1" + "php": "7.3" } }, "require": { diff --git a/composer.lock b/composer.lock index c180d281..2280a5bc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "deed77c6cbecc6d46326aa9f6e3fc79a", + "content-hash": "42a5fecb0cb930ef9fce18ff72ca8800", "packages": [], "packages-dev": [ { @@ -66,7 +66,7 @@ "platform": [], "platform-dev": [], "platform-overrides": { - "php": "7.1" + "php": "7.3" }, - "plugin-api-version": "2.0.0" + "plugin-api-version": "2.1.0" } diff --git a/sonar-project.properties b/sonar-project.properties index f77e715b..b3f17dec 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -29,5 +29,5 @@ sonar.pullrequest.branch=${env.SONAR_PULL_REQUEST_BRANCH} sonar.pullrequest.key=${env.SONAR_PULL_REQUEST_KEY} # Properties specific to language plugins: -sonar.php.coverage.reportPaths=results/clover-phpunit-php7.2-mariadb10.2.xml,results/clover-phpunit-php7.2-mysql8.0.xml,results/clover-phpunit-php7.2-postgres9.4.xml,results/clover-phpunit-php7.2-oracle.xml,results/clover-phpunit-php7.2-sqlite.xml +sonar.php.coverage.reportPaths=results/clover-phpunit-php7.3-mariadb10.2.xml,results/clover-phpunit-php7.3-mysql8.0.xml,results/clover-phpunit-php7.3-postgres9.4.xml,results/clover-phpunit-php7.3-oracle.xml,results/clover-phpunit-php7.3-sqlite.xml sonar.javascript.lcov.reportPaths=results/lcov.info