Merge branch 'master' into update-agent-server-parse

.github/workflows/publish_docker_image.yml

@@ -21,7 +21,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        with:
+          submodules: recursive
 
       - name: Log in to the Container registry
         uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc

.readthedocs.yaml

@@ -0,0 +1,29 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: plugins/fieldmanual/sphinx-docs/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: requirements.txt
+
+# Build all formats (incl. pdf, epub)
+formats: all
+
+# Include all submodules
+submodules:
+  include: all
+  recursive: true

CITATION.cff

@@ -2,7 +2,7 @@ cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 authors:
 - name: "MITRE Corporation"
-title: "CALDERA: A Scalable, Automated Adversary Emulation Platform"
-version: 4.1.0
-date-released: 2022-09-17
+title: "MITRE Caldera: A Scalable, Automated Adversary Emulation Platform"
+version: 4.2.0
+date-released: 2023-06-19
 url: "https://github.com/mitre/caldera"

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:latest
+FROM ubuntu:23.04
 SHELL ["/bin/bash", "-c"]
 
 ARG TZ="UTC"
@@ -12,26 +12,31 @@ ADD . .
 RUN if [ -z "$(ls plugins/stockpile)" ]; then echo "stockpile plugin not downloaded - please ensure you recursively cloned the caldera git repository and try again."; exit 1; fi
 
 RUN apt-get update && \
-    apt-get -y install python3 python3-pip git curl
+    apt-get -y install python3 python3-pip python3-venv git curl golang-go
+
 
 #WIN_BUILD is used to enable windows build in sandcat plugin
 ARG WIN_BUILD=false
 RUN if [ "$WIN_BUILD" = "true" ] ; then apt-get -y install mingw-w64; fi
 
+# Set up python virtualenv
+ENV VIRTUAL_ENV=/opt/venv/caldera
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 # Install pip requirements
 RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Set up config file and disable atomic by default
 RUN python3 -c "import app; import app.utility.config_generator; app.utility.config_generator.ensure_local_config();"; \
     sed -i '/\- atomic/d' conf/local.yml;
 
-# Install golang
-RUN curl -L https://go.dev/dl/go1.17.6.linux-amd64.tar.gz -o go1.17.6.linux-amd64.tar.gz
-RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz;
-ENV PATH="${PATH}:/usr/local/go/bin"
-RUN go version;
-
 # Compile default sandcat agent binaries, which will download basic golang dependencies.
+
+# Install Go dependencies
+WORKDIR /usr/src/app/plugins/sandcat/gocat
+RUN go mod tidy && go mod download
+
 WORKDIR /usr/src/app/plugins/sandcat
 
 # Fix line ending error that can be caused by cloning the project in a Windows environment

README.md

@@ -1,5 +1,5 @@
 [![Release](https://img.shields.io/badge/dynamic/json?color=blue&label=Release&query=tag_name&url=https%3A%2F%2Fapi.github.com%2Frepos%2Fmitre%2Fcaldera%2Freleases%2Flatest)](https://github.com/mitre/caldera/releases/latest)
-[![Testing Status](https://github.com/mitre/caldera/actions/workflows/testing.yml/badge.svg?branch=master)](https://github.com/mitre/caldera/actions/workflows/testing.yml?query=branch%3Amaster)
+[![Testing Status](https://github.com/mitre/caldera/actions/workflows/quality.yml/badge.svg?branch=master)](https://github.com/mitre/caldera/actions/workflows/quality.yml?query=branch%3Amaster)
 [![Security Status](https://github.com/mitre/caldera/actions/workflows/security.yml/badge.svg?branch=master)](https://github.com/mitre/caldera/actions/workflows/security.yml?query=branch%3Amaster)
 [![codecov](https://codecov.io/gh/mitre/caldera/branch/master/graph/badge.svg)](https://codecov.io/gh/mitre/caldera)
 [![Documentation Status](https://readthedocs.org/projects/caldera/badge/?version=stable)](http://caldera.readthedocs.io/?badge=stable)
@@ -30,6 +30,7 @@ These plugins are supported and maintained by the Caldera team.
 - **[Access](https://github.com/mitre/access)** (red team initial access tools and techniques)
 - **[Atomic](https://github.com/mitre/atomic)** (Atomic Red Team project TTPs)
 - **[Builder](https://github.com/mitre/builder)** (dynamically compile payloads)
+- **[Caldera for OT](https://github.com/mitre/caldera-ot)** (ICS/OT capabilities for Caldera)
 - **[Compass](https://github.com/mitre/compass)** (ATT&CK visualizations)
 - **[Debrief](https://github.com/mitre/debrief)** (operations insights)
 - **[Emu](https://github.com/mitre/emu)** (CTID emulation plans)
@@ -79,7 +80,7 @@ Next, install the PIP requirements:
 ```Bash
 pip3 install -r requirements.txt
 ```
-**Super-power your Caldea server installation! [Install GoLang (1.17+)](https://go.dev/doc/install)**
+**Super-power your Caldera server installation! [Install GoLang (1.19+)](https://go.dev/doc/install)**
 
 Finally, start the server.
 ```Bash

app/api/v2/managers/operation_api_manager.py

@@ -87,6 +87,8 @@ async def update_operation_link(self, operation_id: str, link_id: str, link_data
             if not link.is_valid_status(link_status):
                 raise JsonHttpBadRequest(f'Cannot update link {link_id} due to invalid link status.')
             link.status = link_status
+            if link.can_ignore():
+                operation.add_ignored_link(link.id)
         return link.display
 
     async def create_potential_link(self, operation_id: str, data: dict, access: BaseWorld.Access):

app/contacts/contact_tcp.py

@@ -85,6 +85,7 @@ async def send(self, session_id: int, cmd: str, timeout: int = 60) -> Tuple[int,
         try:
             conn = next(i.connection for i in self.sessions if i.id == int(session_id))
             conn.send(str.encode(' '))
+            time.sleep(0.01)
             conn.send(str.encode('%s\n' % cmd))
             response = await self._attempt_connection(session_id, conn, timeout=timeout)
             response = json.loads(response)

app/objects/c_ability.py

@@ -13,21 +13,21 @@
 
 class AbilitySchema(ma.Schema):
     ability_id = ma.fields.String()
-    tactic = ma.fields.String(missing=None)
-    technique_name = ma.fields.String(missing=None)
-    technique_id = ma.fields.String(missing=None)
-    name = ma.fields.String(missing=None)
-    description = ma.fields.String(missing=None)
+    tactic = ma.fields.String(load_default=None)
+    technique_name = ma.fields.String(load_default=None)
+    technique_id = ma.fields.String(load_default=None)
+    name = ma.fields.String(load_default=None)
+    description = ma.fields.String(load_default=None)
     executors = ma.fields.List(ma.fields.Nested(ExecutorSchema))
-    requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None)
-    privilege = ma.fields.String(missing=None)
-    repeatable = ma.fields.Bool(missing=None)
-    buckets = ma.fields.List(ma.fields.String(), missing=None)
+    requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), load_default=None)
+    privilege = ma.fields.String(load_default=None)
+    repeatable = ma.fields.Bool(load_default=None)
+    buckets = ma.fields.List(ma.fields.String(), load_default=None)
     additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())
-    access = ma.fields.Nested(AccessSchema, missing=None)
-    singleton = ma.fields.Bool(missing=None)
-    plugin = ma.fields.String(missing=None)
-    delete_payload = ma.fields.Bool(missing=None)
+    access = ma.fields.Nested(AccessSchema, load_default=None)
+    singleton = ma.fields.Bool(load_default=None)
+    plugin = ma.fields.String(load_default=None)
+    delete_payload = ma.fields.Bool(load_default=None)
 
     @ma.pre_load
     def fix_id(self, data, **_):

app/objects/c_adversary.py

@@ -18,7 +18,7 @@ class AdversarySchema(ma.Schema):
     objective = ma.fields.String()
     tags = ma.fields.List(ma.fields.String(), allow_none=True)
     has_repeatable_abilities = ma.fields.Boolean(dump_only=True)
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.pre_load
     def fix_id(self, adversary, **_):

app/objects/c_operation.py

@@ -50,7 +50,7 @@ class OperationSchema(ma.Schema):
     visibility = ma.fields.Integer()
     objective = ma.fields.Nested(ObjectiveSchema())
     use_learning_parsers = ma.fields.Boolean()
-    group = ma.fields.String(missing='')
+    group = ma.fields.String(load_default='')
     source = ma.fields.Nested(SourceSchema())
 
     @ma.pre_load()
@@ -234,7 +234,7 @@ async def wait_for_links_completion(self, link_ids):
         for link_id in link_ids:
             link = [link for link in self.chain if link.id == link_id][0]
             if link.can_ignore():
-                self.ignored_links.add(link.id)
+                self.add_ignored_link(link.id)
             member = [member for member in self.agents if member.paw == link.paw][0]
             while not (link.finish or link.can_ignore()):
                 await asyncio.sleep(5)
@@ -256,6 +256,9 @@ async def is_finished(self):
     def link_status(self):
         return -3 if self.autonomous else -1
 
+    def add_ignored_link(self, link_id):
+        self.ignored_links.add(link_id)
+
     async def active_agents(self):
         active = []
         for agent in self.agents:
@@ -272,7 +275,7 @@ async def get_skipped_abilities_by_agent(self, data_svc):
         for agent in self.agents:
             agent_skipped = defaultdict(dict)
             agent_executors = agent.executors
-            agent_ran = set([link.ability.ability_id for link in self.chain if link.paw == agent.paw])
+            agent_ran = set([link.ability.ability_id for link in self.chain if link.paw == agent.paw and link.finish])
             for ab in abilities_by_agent[agent.paw]['all_abilities']:
                 skipped = self._check_reason_skipped(agent=agent, ability=ab, agent_executors=agent_executors,
                                                      op_facts=[f.trait for f in await self.all_facts()],
@@ -438,8 +441,13 @@ async def _unfinished_links_for_agent(self, paw):
 
     async def _get_all_possible_abilities_by_agent(self, data_svc):
         abilities = {'all_abilities': [ab for ab_id in self.adversary.atomic_ordering
-                     for ab in await data_svc.locate('abilities', match=dict(ability_id=ab_id))]}
-        return {a.paw: abilities for a in self.agents}
+                                       for ab in await data_svc.locate('abilities', match=dict(ability_id=ab_id))]}
+        abilities_by_agent = {a.paw: abilities for a in self.agents}
+        for link in self.chain:
+            if link.ability.ability_id not in self.adversary.atomic_ordering:
+                matching_abilities = await data_svc.locate('abilities', match=dict(ability_id=link.ability.ability_id))
+                abilities_by_agent[link.paw]['all_abilities'].extend(matching_abilities)
+        return abilities_by_agent
 
     def _check_reason_skipped(self, agent, ability, op_facts, state, agent_executors, agent_ran):
         if ability.ability_id in agent_ran:

app/objects/c_planner.py

@@ -16,7 +16,7 @@ class PlannerSchema(ma.Schema):
     stopping_conditions = ma.fields.List(ma.fields.Nested(FactSchema()))
     ignore_enforcement_modules = ma.fields.List(ma.fields.String())
     allow_repeatable_abilities = ma.fields.Boolean()
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.post_load()
     def build_planner(self, data, **kwargs):

app/objects/c_source.py

@@ -33,7 +33,7 @@ class SourceSchema(ma.Schema):
     rules = ma.fields.List(ma.fields.Nested(RuleSchema))
     adjustments = ma.fields.List(ma.fields.Nested(AdjustmentSchema))
     relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema))
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.pre_load
     def fix_adjustments(self, in_data, **_):
@@ -60,17 +60,15 @@ def _fix_loaded_object_origins(input_data):
         :param input_data: A 'source' dictionary
         :return: input_data with updated facts/relationships (patched in place)
         """
-        if 'facts' in input_data:
-            for y in input_data['facts']:
-                y['origin_type'] = OriginType.IMPORTED.name
-                y['source'] = input_data['id']
-        if 'relationships' in input_data:
-            for y in input_data['relationships']:
-                y['source']['origin_type'] = OriginType.IMPORTED.name
-                y['source']['source'] = input_data['id']
-                if 'target' in y:
-                    y['target']['origin_type'] = OriginType.IMPORTED.name
-                    y['target']['source'] = input_data['id']
+        for y in input_data.get('facts', []):
+            y['origin_type'] = OriginType.IMPORTED.name
+            y['source'] = input_data['id']
+        for y in input_data.get('relationships', []):
+            y['source']['origin_type'] = OriginType.IMPORTED.name
+            y['source']['source'] = input_data['id']
+            if y.get('target'):
+                y['target']['origin_type'] = OriginType.IMPORTED.name
+                y['target']['source'] = input_data['id']
 
 
 class Source(FirstClassObjectInterface, BaseObject):

app/objects/secondclass/c_executor.py

@@ -7,15 +7,15 @@
 
 
 class ExecutorSchema(ma.Schema):
-    name = ma.fields.String(missing=None)
-    platform = ma.fields.String(missing=None)
-    command = ma.fields.String(missing=None)
-    code = ma.fields.String(missing=None)
-    language = ma.fields.String(missing=None)
-    build_target = ma.fields.String(missing=None)
+    name = ma.fields.String(load_default=None)
+    platform = ma.fields.String(load_default=None)
+    command = ma.fields.String(load_default=None)
+    code = ma.fields.String(load_default=None)
+    language = ma.fields.String(load_default=None)
+    build_target = ma.fields.String(load_default=None)
     payloads = ma.fields.List(ma.fields.String())
     uploads = ma.fields.List(ma.fields.String())
-    timeout = ma.fields.Int(missing=60)
+    timeout = ma.fields.Int(load_default=60)
     parsers = ma.fields.List(ma.fields.Nested(ParserSchema()))
     cleanup = ma.fields.List(ma.fields.String())
     variations = ma.fields.List(ma.fields.Nested(VariationSchema()))

app/objects/secondclass/c_instruction.py

@@ -12,7 +12,7 @@ class InstructionSchema(ma.Schema):
     payloads = ma.fields.List(ma.fields.String())
     deadman = ma.fields.Boolean()
     uploads = ma.fields.List(ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()))
-    delete_payload = ma.fields.Bool(missing=None)
+    delete_payload = ma.fields.Bool(load_default=None)
 
     @ma.post_load
     def build_instruction(self, data, **_):

app/objects/secondclass/c_link.py

@@ -25,15 +25,15 @@ class LinkSchema(ma.Schema):
     class Meta:
         unknown = ma.EXCLUDE
 
-    id = ma.fields.String(missing='')
+    id = ma.fields.String(load_default='')
     paw = ma.fields.String()
     command = ma.fields.String()
     plaintext_command = ma.fields.String()
-    status = ma.fields.Integer(missing=-3)
-    score = ma.fields.Integer(missing=0)
-    jitter = ma.fields.Integer(missing=0)
+    status = ma.fields.Integer(load_default=-3)
+    score = ma.fields.Integer(load_default=0)
+    jitter = ma.fields.Integer(load_default=0)
     decide = ma.fields.DateTime(format=BaseObject.TIME_FORMAT)
-    pin = ma.fields.Integer(missing=0)
+    pin = ma.fields.Integer(load_default=0)
     pid = ma.fields.String()
     facts = ma.fields.List(ma.fields.Nested(FactSchema()))
     relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema()))
@@ -43,12 +43,12 @@ class Meta:
     finish = ma.fields.String()
     ability = ma.fields.Nested(AbilitySchema())
     executor = ma.fields.Nested(ExecutorSchema())
-    cleanup = ma.fields.Integer(missing=0)
+    cleanup = ma.fields.Integer(load_default=0)
     visibility = ma.fields.Nested(VisibilitySchema())
-    host = ma.fields.String(missing=None)
+    host = ma.fields.String(load_default=None)
     output = ma.fields.String()
     deadman = ma.fields.Boolean()
-    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, missing=None)
+    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, load_default=None)
 
     @ma.pre_load()
     def fix_ability(self, link, **_):

app/objects/secondclass/c_parserconfig.py

@@ -9,8 +9,8 @@ class Meta:
         unknown = ma.INCLUDE
 
     source = ma.fields.String()
-    edge = ma.fields.String(missing=None)
-    target = ma.fields.String(missing=None)
+    edge = ma.fields.String(load_default=None)
+    target = ma.fields.String(load_default=None)
     custom_parser_vals = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())
 
     @ma.pre_load

app/objects/secondclass/c_result.py

@@ -10,7 +10,7 @@ class ResultSchema(ma.Schema):
     exit_code = ma.fields.String()
     pid = ma.fields.String()
     status = ma.fields.String()
-    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, missing=None)
+    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, load_default=None)
 
     @ma.post_load
     def build_result(self, data, **_):