build.gradle

buildscript {
    repositories {
        mavenCentral()
    }
    dependencies {
        classpath 'de.thetaphi:forbiddenapis:3.8'
    }
}
plugins {
    id 'maven-publish'
    // https://github.com/researchgate/gradle-release
    id 'net.researchgate.release' version '3.0.2'
    id 'io.codearte.nexus-staging' version '0.30.0'
}

apply plugin: 'java'
apply plugin: 'de.thetaphi.forbiddenapis'
apply plugin: 'jacoco'
apply plugin: 'signing'

java {
  toolchain {
	  sourceCompatibility = JavaVersion.VERSION_17
  }
}
group = 'org.dstadler'
archivesBaseName = 'commoncrawldownload'

repositories {
    mavenCentral()
}

forbiddenApis {
    suppressAnnotations = ['org.dstadler.commons.util.SuppressForbidden']
    bundledSignatures = [ 'jdk-reflection', 'commons-io-unsafe-2.15.1', 'jdk-internal' ]
    signaturesFiles += files('config/forbidden-apis/http-signatures.txt')
}
forbiddenApisMain {
    // 'jdk-unsafe', 'jdk-system-out'
    bundledSignatures += [ 'jdk-deprecated', 'jdk-internal', 'jdk-non-portable' ]
    signaturesFiles += files('config/forbidden-apis/forbidden.signatures.txt')
}

dependencies {
    implementation 'org.dstadler:commons-dost:1.3.4'
    implementation 'org.apache.httpcomponents:httpclient:4.5.14'
    implementation 'org.apache.commons:commons-lang3:3.17.0'
    implementation 'commons-io:commons-io:2.18.0'

    // version '0.20.2-cdh3u4' is referenced by webarchive-commons, but "vanished"...
    implementation 'org.apache.hadoop:hadoop-core:0.20.205.0'
    implementation 'org.netpreserve.commons:webarchive-commons:1.1.9'
    implementation 'com.fasterxml.jackson.core:jackson-core:2.18.1'

    // for commoncrawl-source
    implementation 'org.jsoup:jsoup:1.15.3'

    // for @Nullable
    implementation 'com.google.code.findbugs:jsr305:3.0.2'
    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.11.4'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.11.4'
    testRuntimeOnly 'org.junit.platform:junit-platform-launcher:1.11.4'
    testImplementation 'org.dstadler:commons-test:1.0.0.21'
}

wrapper {
    gradleVersion = '8.5'
}

test {
    systemProperties = System.properties as Map<String, ?>

    // enable to show standard out and standard error of the test JVM(s) on the console
    // testLogging.showStandardStreams = true

    // http://forums.gradle.org/gradle/topics/jacoco_related_failure_in_multiproject_build
    systemProperties['user.dir'] = workingDir

    useJUnitPlatform()
}

jacoco {
    toolVersion = '0.8.12'
}

jacocoTestReport {
    reports {
        xml.required = true
    }
}

task sourcesJar(type: Jar) {
    archiveClassifier = 'sources'
    from sourceSets.main.allJava
}

task javadocJar(type: Jar) {
    archiveClassifier = 'javadoc'
    from javadoc
}

java {
    withJavadocJar()
    withSourcesJar()
}

release {
    git {
        requireBranch.set('master')
    }
}

publishing {
    publications {
        mavenJava(MavenPublication) {
            artifactId = archivesBaseName
            from components.java

            pom {
                name = 'CommonCrawl Document Download'
                description = 'Helper tools to download data from CommonCrawl archives'
                url = 'https://github.com/centic9/CommonCrawlDocumentDownload'
                licenses {
                    license {
                        name = 'BSD 2-Clause "Simplified" License'
                        url = 'https://github.com/centic9/CommonCrawlDocumentDownload/blob/master/LICENSE.md'
                    }
                }
                developers {
                    developer {
                        id = 'centic9'
                        name = 'Dominik Stadler'
                    }
                }
                scm {
                    connection = 'scm:git@github.com:centic9/CommonCrawlDocumentDownload'
                    developerConnection = 'scm:git@github.com:centic9/CommonCrawlDocumentDownload'
                    url = 'https://github.com/centic9/CommonCrawlDocumentDownload'
                }
            }
        }
    }
    repositories {
        maven {
            // change to point to your repo, e.g. http://my.org/repo
            //url = layout.buildDirectory.dir('repo')
            url = 'https://oss.sonatype.org/service/local/staging/deploy/maven2/'
            credentials(PasswordCredentials)
            //snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots/')
        }
    }
}

signing {
    sign publishing.publications.mavenJava
}

afterReleaseBuild.dependsOn publish

task lookupURLs(type:JavaExec,dependsOn: compileJava) {
    description = 'Reads the current Common Crawl URL index data and extracts all URLs for interesting mime-types or file extensions'

    jvmArgs = [
            '-Xmx128m'
    ]

    mainClass = 'org.dstadler.commoncrawl.index.DownloadURLIndex'
    classpath = sourceSets.main.runtimeClasspath
}


task downloadOldIndex(type:JavaExec,dependsOn: compileJava) {
    description = 'Reading blocks at beginning or the last stored block-number and download binary data from the common crawl archives'

    jvmArgs = [
            '-Xmx128m'
    ]

    mainClass = 'org.dstadler.commoncrawl.oldindex.ReadAndDownload'
    classpath = sourceSets.main.runtimeClasspath
}

task downloadDocuments(type:JavaExec,dependsOn: compileJava) {
    description = 'Uses the URLs listed in commoncrawl.txt to download the documents from the Common Crawl'

    jvmArgs = [
            '-Xmx128m'
    ]

    mainClass = 'org.dstadler.commoncrawl.index.DownloadFromCommonCrawl'
    classpath = sourceSets.main.runtimeClasspath
}

task deduplicate(type:JavaExec,dependsOn: compileJava) {
    description = 'Compares files by size and hash and moves away identical files into a backup-directory'

    jvmArgs = [
            '-Xmx1024m'
    ]

    mainClass = 'org.dstadler.commoncrawl.Deduplicate'
    classpath = sourceSets.main.runtimeClasspath
}