Skip to content

Commit

Permalink
Merge branch 'master' into multi_comment_email
Browse files Browse the repository at this point in the history
Signed-off-by: Diego Tavares da Silva <[email protected]>
  • Loading branch information
DiegoTavares authored Dec 7, 2022
2 parents 76ce396 + 529d606 commit 75f07db
Show file tree
Hide file tree
Showing 103 changed files with 1,680 additions and 943 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ venv*/
coverage.xml
htmlcov/
/.env
.envrc
2 changes: 1 addition & 1 deletion VERSION.in
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.18
0.21
62 changes: 11 additions & 51 deletions cueadmin/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,43 +1,22 @@
# -----------------
# BUILD
# -----------------
FROM centos:7 as build

# First line after FROM should be unique to avoid --cache-from weirdness.
RUN echo "CueAdmin build stage"
FROM python:3.9.14

WORKDIR /src

RUN yum -y install \
epel-release \
gcc \
python-devel

RUN yum -y install \
python-pip \
python36 \
python36-devel \
python36-pip

RUN python -m pip install --upgrade 'pip<21'
RUN python3.6 -m pip install --upgrade pip

RUN python -m pip install --upgrade 'setuptools<45'
RUN python3.6 -m pip install --upgrade setuptools
RUN python3 -m pip install --upgrade pip
RUN python3 -m pip install --upgrade setuptools

COPY LICENSE ./
COPY requirements.txt ./

RUN python -m pip install -r requirements.txt
RUN python3.6 -m pip install -r requirements.txt
RUN python3 -m pip install -r requirements.txt

COPY proto/ ./proto
COPY pycue/README.md ./pycue/
COPY pycue/setup.py ./pycue/
COPY pycue/opencue ./pycue/opencue
COPY pycue/FileSequence ./pycue/FileSequence

RUN python -m grpc_tools.protoc \
RUN python3 -m grpc_tools.protoc \
-I=./proto \
--python_out=./pycue/opencue/compiled_proto \
--grpc_python_out=./pycue/opencue/compiled_proto \
Expand All @@ -55,32 +34,13 @@ COPY cueadmin/cueadmin ./cueadmin/cueadmin
COPY VERSION.in VERSIO[N] ./
RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION

RUN cd pycue && python setup.py install

RUN cd pycue && python3.6 setup.py install

# TODO(bcipriano) Lint the code here. (Issue #78)

RUN cd cueadmin && python setup.py test

RUN cd cueadmin && python3.6 setup.py test
RUN cd pycue && python3 setup.py install
RUN cd cueadmin && python3 setup.py test

RUN cp LICENSE requirements.txt VERSION cueadmin/

RUN versioned_name="cueadmin-$(cat ./VERSION)-all" \
&& mv cueadmin $versioned_name \
&& tar -cvzf $versioned_name.tar.gz $versioned_name/*


# -----------------
# RUN
# -----------------
FROM centos:7

# First line after FROM should be unique to avoid --cache-from weirdness.
RUN echo "CueAdmin runtime stage"

WORKDIR /opt/opencue

COPY --from=build /src/cueadmin-*-all.tar.gz ./

&& mv cueadmin "${versioned_name}" \
&& tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \
&& mkdir -p /opt/opencue \
&& cp "${versioned_name}.tar.gz" /opt/opencue/
16 changes: 3 additions & 13 deletions cuebot/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
# -----------------
# BUILD
# -----------------
FROM gradle:6.0.1-jdk11 AS build
FROM gradle:6.0.1-jdk13 AS build

USER gradle

# First line after FROM should be unique to avoid --cache-from weirdness.
RUN echo "Cuebot build stage"

COPY --chown=gradle:gradle ./cuebot /home/gradle/cuebot/
COPY --chown=gradle:gradle ./proto /home/gradle/proto/

WORKDIR /home/gradle/cuebot

RUN gradle build --stacktrace
RUN gradle build --info --stacktrace

COPY --chown=gradle:gradle VERSION.in VERSIO[N] ./
RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION
Expand All @@ -25,12 +22,8 @@ RUN mv ./build/libs/cuebot.jar ./build/libs/cuebot-$(cat ./VERSION)-all.jar
# -----------------
FROM jc21/rpmbuild-centos7:latest AS rpm

# Random first line after from

USER rpmbuilder

RUN echo "Cuebot RPM Stage"

COPY --chown=rpmbuilder:rpmbuilder LICENSE ./

COPY --from=build \
Expand All @@ -49,10 +42,7 @@ RUN chmod +x create_rpm.sh && ./create_rpm.sh cuebot "$(cat VERSION)"
# -----------------
# RUN
# -----------------
FROM openjdk:11-jre-slim-buster

# First line after FROM should be unique to avoid --cache-from weirdness.
RUN echo "Cuebot runtime stage"
FROM openjdk:18-slim-buster

ARG CUEBOT_GRPC_CUE_PORT=8443
ARG CUEBOT_GRPC_RQD_PORT=8444
Expand Down
8 changes: 4 additions & 4 deletions cuebot/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ dependencies {
compile group: 'com.google.guava', name: 'guava', version: '26.0-android'
compile group: 'com.sun.mail', name: 'mailapi', version: '1.5.4'
compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
compile group: 'io.grpc', name: 'grpc-all', version: '1.36.2'
compile group: 'io.grpc', name: 'grpc-all', version: '1.47.0'
compile group: 'org.apache.activemq', name: 'activemq-pool', version: activemqVersion
compile group: 'org.apache.velocity', name: 'velocity', version: '1.7'
compile group: 'org.jdom', name: 'jdom', version: '1.1.3'
Expand All @@ -54,7 +54,7 @@ dependencies {
compile group: 'org.springframework', name: 'spring-jms'
compile group: 'org.quartz-scheduler', name: 'quartz', version: '2.2.1', { exclude group: 'c3p0', module: 'c3p0' }
compile group: 'org.postgresql', name: 'postgresql', version: '42.2.2'
compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.13.0'
compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.2'
compile group: 'log4j', name: 'log4j', version: '1.2.17'
compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26'

Expand All @@ -81,12 +81,12 @@ compileTestJava {
protobuf {
protoc {
// The protoc compiler
artifact = 'com.google.protobuf:protoc:3.13.0'
artifact = 'com.google.protobuf:protoc:3.21.2'
}
plugins {
grpc {
// Generate gRPC stubs.
artifact = 'io.grpc:protoc-gen-grpc-java:1.7.0'
artifact = 'io.grpc:protoc-gen-grpc-java:1.47.0'
}
}
generateProtoTasks {
Expand Down
34 changes: 32 additions & 2 deletions cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,48 @@
*/



package com.imageworks.spcue;

import java.util.Arrays;
import java.util.Optional;
import java.util.stream.Stream;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class CuebotApplication extends SpringApplication {
private static String[] checkArgs(String[] args) {
Optional<String> deprecatedFlag = Arrays.stream(args)
.filter(arg -> arg.startsWith("--log.frame-log-root=")).findFirst();
if (deprecatedFlag.isPresent()) {
// Log a deprecation warning.
Logger warning_logger = Logger.getLogger(CuebotApplication.class);
warning_logger.warn("`--log.frame-log-root` is deprecated and will be removed in an " +
"upcoming release. It has been replaced with `--log.frame-log-root.default_os`. " +
"See opencue.properties for details on OpenCue's new OS-dependent root directories.");
// If new flags are not present, swap in the value provided using the new flag.
// If the new flags are already present, don't do anything.
Optional<String> newFlags = Arrays.stream(args)
.filter(arg -> arg.startsWith("--log.frame-log-root.")).findAny();
if (!newFlags.isPresent()) {
String fixedFlag = "--log.frame-log-root.default_os="
+ StringUtils.substringAfter(deprecatedFlag.get(), "=");
args = Stream.concat(
Arrays.stream(args).filter(arg -> !arg.startsWith("--log.frame-log-root=")),
Stream.of(fixedFlag))
.toArray(String[]::new);
}
}
return args;
}

public static void main(String[] args) {
// Cuebot startup
SpringApplication.run(CuebotApplication.class, args);
String[] filteredArgs = checkArgs(args);
SpringApplication.run(CuebotApplication.class, filteredArgs);
}
}

2 changes: 2 additions & 0 deletions cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,7 @@ public class ServiceEntity extends Entity {

public int timeout_llu = 0;

public long minMemoryIncrease = Dispatcher.MINIMUM_MEMORY_INCREASE;

}

Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import java.util.List;
import java.util.Map;

import com.imageworks.spcue.dispatcher.AbstractDispatcher;
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.jdbc.core.support.JdbcDaoSupport;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public ServiceEntity mapRow(ResultSet rs, int rowNum) throws SQLException {
s.tags = splitTags(rs.getString("str_tags"));
s.timeout = rs.getInt("int_timeout");
s.timeout_llu = rs.getInt("int_timeout_llu");
s.minMemoryIncrease = rs.getLong("int_min_memory_increase");
return s;
}
};
Expand All @@ -89,6 +90,7 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum)
s.showId = rs.getString("pk_show");
s.timeout = rs.getInt("int_timeout");
s.timeout_llu = rs.getInt("int_timeout_llu");
s.minMemoryIncrease = rs.getLong("int_min_memory_increase");
return s;
}
};
Expand All @@ -106,7 +108,8 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum)
"service.int_gpu_mem_min," +
"service.str_tags, " +
"service.int_timeout, " +
"service.int_timeout_llu " +
"service.int_timeout_llu, " +
"service.int_min_memory_increase " +
"FROM " +
"service ";

Expand All @@ -131,6 +134,7 @@ public ServiceEntity get(String id) {
"show_service.str_tags," +
"show_service.int_timeout," +
"show_service.int_timeout_llu," +
"show_service.int_min_memory_increase," +
"show.pk_show " +
"FROM " +
"show_service," +
Expand Down Expand Up @@ -180,8 +184,9 @@ public boolean isOverridden(String service, String show) {
"int_gpu_mem_min," +
"str_tags," +
"int_timeout," +
"int_timeout_llu " +
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?)";
"int_timeout_llu, " +
"int_min_memory_increase " +
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)";

@Override
public void insert(ServiceEntity service) {
Expand All @@ -191,7 +196,8 @@ public void insert(ServiceEntity service) {
service.maxCores, service.minMemory,
service.minGpus, service.maxGpus, service.minGpuMemory,
StringUtils.join(service.tags.toArray(), " | "),
service.timeout, service.timeout_llu);
service.timeout, service.timeout_llu,
service.minMemoryIncrease);
}

private static final String INSERT_SERVICE_WITH_SHOW =
Expand All @@ -210,8 +216,9 @@ public void insert(ServiceEntity service) {
"int_gpu_mem_min," +
"str_tags," +
"int_timeout," +
"int_timeout_llu " +
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)";
"int_timeout_llu, " +
"int_min_memory_increase " +
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)";

@Override
public void insert(ServiceOverrideEntity service) {
Expand All @@ -220,7 +227,7 @@ public void insert(ServiceOverrideEntity service) {
service.showId, service.name, service.threadable,
service.minCores, service.maxCores, service.minMemory,
service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags),
service.timeout, service.timeout_llu);
service.timeout, service.timeout_llu, service.minMemoryIncrease);
}

private static final String UPDATE_SERVICE =
Expand All @@ -237,7 +244,8 @@ service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags),
"int_gpu_mem_min=?," +
"str_tags=?," +
"int_timeout=?," +
"int_timeout_llu=? " +
"int_timeout_llu=?, " +
"int_min_memory_increase=? " +
"WHERE " +
"pk_service = ?";

Expand All @@ -246,7 +254,8 @@ public void update(ServiceEntity service) {
getJdbcTemplate().update(UPDATE_SERVICE, service.name,
service.threadable, service.minCores, service.maxCores,
service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags),
service.timeout, service.timeout_llu, service.getId());
service.timeout, service.timeout_llu, service.minMemoryIncrease,
service.getId());
}

private static final String UPDATE_SERVICE_WITH_SHOW =
Expand All @@ -263,7 +272,8 @@ service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinT
"int_gpu_mem_min=?," +
"str_tags=?," +
"int_timeout=?," +
"int_timeout_llu=? " +
"int_timeout_llu=?, " +
"int_min_memory_increase=? " +
"WHERE " +
"pk_show_service = ?";

Expand All @@ -272,7 +282,8 @@ public void update(ServiceOverrideEntity service) {
getJdbcTemplate().update(UPDATE_SERVICE_WITH_SHOW, service.name,
service.threadable, service.minCores, service.maxCores,
service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags),
service.timeout, service.timeout_llu, service.getId());
service.timeout, service.timeout_llu, service.minMemoryIncrease,
service.getId());
}

@Override
Expand Down
Loading

0 comments on commit 75f07db

Please sign in to comment.