diff --git a/analyzer/QUIC.spicy b/analyzer/QUIC.spicy index d7a3cfb..e7276f2 100644 --- a/analyzer/QUIC.spicy +++ b/analyzer/QUIC.spicy @@ -7,34 +7,37 @@ import spicy; import zeek; # The interface to the C++ code that handles the decryption of the INITIAL packet payload using well-known keys -public function decrypt_crypto_payload(entire_packet: bytes, connection_id: bytes, encrypted_offset: uint64, payload_offset: uint64, from_client: bool): bytes &cxxname="decrypt_crypto_payload"; +public function decrypt_crypto_payload( + all_data: bytes, + connection_id: bytes, + encrypted_offset: uint64, + payload_offset: uint64, + from_client: bool +): bytes &cxxname="QUIC_decrypt_crypto_payload"; ############## ## Context - tracked in one connection ############## -# Should we buffer the packet? -function should_buffer(context: ConnectionIDInfo, is_client: bool): bool { - if ( is_client ) - return ! context.client_initial_processed; - - return context.client_initial_processed - && |context.initial_destination_conn_id| > 0 - && ! context.server_initial_processed; -} - # Can we decrypt? -function can_decrypt(long_header: LongHeaderPacket): bool { +function can_decrypt(long_header: LongHeaderPacket, context: ConnectionIDInfo, is_client: bool): bool { if ( long_header.first_byte.packet_type != LongPacketType::INITIAL ) return False; # decrypt_crypto_payload() has known secrets for version 1, nothing else. - if ( long_header.version == 0x00000001 ) - return True; + if ( long_header.version != 0x00000001 ) + return False; - return False; + if ( is_client ) + return ! context.client_initial_processed; + + # This is the responder, can only decrypt if we have an initial + # destination_id from the client + return context.client_initial_processed + && |context.initial_destination_conn_id| > 0 + && ! context.server_initial_processed; } type ConnectionIDInfo = struct { @@ -128,17 +131,6 @@ type FrameType = enum { # Helper units ############## -# Used to peek into the next byte and determine if it's a long or short packet -public type InitialByte = unit { - initialbyte: bitfield(8) { - header_form: 7 &convert=cast(cast($$)); - long_packet_type: 4..5 &convert=cast(cast($$)); - }; - on %done { - self.backtrack(); - } -}; - type VariableLengthInteger = unit { var bytes_to_parse: uint64; var result: uint64; @@ -167,21 +159,6 @@ type VariableLengthInteger = unit { # Generic units ############## -# Used to capture all data form the entire frame. May be inefficient, but works for now. -# This is passed to the decryption function, as this function needs both the header and the payload -# Performs a backtrack() at the end -type AllData = unit { - var data: bytes; - - : bytes &eod { - self.data = $$; - } - - on %done { - self.backtrack(); - } -}; - public type LongHeaderPacket = unit { var encrypted_offset: uint64; var payload_length: uint64; @@ -229,7 +206,11 @@ public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sin crypto_sink.write(self.c.cryptodata, self.c.offset.result); } FrameType::CONNECTION_CLOSE1 -> : ConnectionClosePayload(header); +@if SPICY_VERSION >= 10800 + FrameType::PADDING -> : skip /\x00*/; # eat the padding +@else FrameType::PADDING -> : /\x00*/; # eat the padding +@endif FrameType::PING -> : void; * -> : void { throw "unhandled frame type %s in %s" % (self.frame_type, header.first_byte.packet_type); @@ -237,11 +218,6 @@ public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sin }; }; -# TODO: investigate whether we can do something useful with this -public type EncryptedLongPacketPayload = unit { - payload: bytes &eod; -}; - type CRYPTOPayload = unit(from_client: bool) { offset: VariableLengthInteger; length: VariableLengthInteger; @@ -292,19 +268,31 @@ type InitialPacket = unit(header: LongHeaderPacket) { # includes the packet number field, but we # do not know its length yet. We need the # payload for sampling, however. +@if SPICY_VERSION >= 10800 + payload: skip bytes &size=self.length.result; +@else payload: bytes &size=self.length.result; +@endif }; type ZeroRTTPacket = unit(header: LongHeaderPacket) { var header: LongHeaderPacket = header; length: VariableLengthInteger; +@if SPICY_VERSION >= 10800 + payload: skip bytes &size=self.length.result; +@else payload: bytes &size=self.length.result; +@endif }; type HandshakePacket = unit(header: LongHeaderPacket) { var header: LongHeaderPacket = header; length: VariableLengthInteger; +@if SPICY_VERSION >= 10800 + payload: skip bytes &size=self.length.result; +@else payload: bytes &size=self.length.result; +@endif }; @@ -340,7 +328,20 @@ public type ShortHeader = unit(dest_conn_id_length: uint8) { # TODO: investigate whether we can parse something useful out of this public type ShortPacketPayload = unit { +@if SPICY_VERSION >= 10800 + payload: skip bytes &eod; +@else payload: bytes &eod; +@endif +}; + +# TODO: investigate whether we can do something useful with this +public type EncryptedLongPacketPayload = unit { +@if SPICY_VERSION >= 10800 + payload: skip bytes &eod; +@else + payload: bytes &eod; +@endif }; # Buffer all crypto messages (which might be fragmented and unordered) @@ -361,10 +362,9 @@ type CryptoBuffer = unit() { # A UDP datagram contains one or more QUIC packets. ############## type Packet = unit(from_client: bool, context: ConnectionIDInfo&) { - var header_form: HeaderForm; - var long_packet_type: LongPacketType; var decrypted_data: bytes; var full_packet: bytes; + var start: iterator; sink crypto_sink; var crypto_buffer: CryptoBuffer&; @@ -376,28 +376,29 @@ type Packet = unit(from_client: bool, context: ConnectionIDInfo&) { context.ssl_handle = zeek::protocol_handle_get_or_create("SSL"); } @else - if ( ! context.did_ssl_begin ) - { + if ( ! context.did_ssl_begin ) { zeek::protocol_begin("SSL"); context.did_ssl_begin = True; - } + } @endif + + self.start = self.input(); } - # Peek into the header to check if it's a SHORT or LONG header - # and get the LONG packet type. - : InitialByte &try { - self.header_form = $$.initialbyte.header_form; - self.long_packet_type = $$.initialbyte.long_packet_type; + # Peek into the first byte and determine the header type. + first_byte: bitfield(8) { + header_form: 7 &convert=HeaderForm($$); + }; + + # TODO: Consider bitfield based look-ahead-parsing in the switch below + # to avoid this rewinding here. It's a hack. + : void { + self.set_input(self.start); # rewind } - # Capture all the packet bytes if we're still have a chance of decrypting the INITIAL PACKETS - fpack: AllData &try if ( self.header_form == HeaderForm::LONG && - self.long_packet_type == LongPacketType::INITIAL && - should_buffer(context, from_client) ); # Depending on the header, parse it and update the src/dest ConnectionID's - switch ( self.header_form ) { + switch ( self.first_byte.header_form ) { HeaderForm::SHORT -> short_header: ShortHeader(context.client_cid_len); HeaderForm::LONG -> long_header: LongHeaderPacket { # For now, only allow a change of src/dest ConnectionID's for INITIAL packets. @@ -418,75 +419,77 @@ type Packet = unit(from_client: bool, context: ConnectionIDInfo&) { context.did_ssl_begin = False; @endif } + } + }; - if ( can_decrypt(self.long_header) && should_buffer(context, from_client) ) { - - # Initialize sink/buffer for accumulating CRYPTO frames. - # - # TODO: Attach this to the context instead of unit. - self.crypto_buffer = new CryptoBuffer(); - self.crypto_sink.connect(self.crypto_buffer); - - if ( from_client ) { - context.server_cid_len = self.long_header.dest_conn_id_len; - context.client_cid_len = self.long_header.src_conn_id_len; - - # This means that here, we can try to decrypt the initial packet! - # All data is accessible via the `long_header` unit - - self.decrypted_data = decrypt_crypto_payload(self.fpack.data, - self.long_header.dest_conn_id, - self.long_header.encrypted_offset, - self.long_header.payload_length, - from_client); - - # Set this to be the seed for the decryption - if ( |context.initial_destination_conn_id| == 0 ) { - context.initial_destination_conn_id = self.long_header.dest_conn_id; - } - - } else { - context.server_cid_len = self.long_header.src_conn_id_len; - context.client_cid_len = self.long_header.dest_conn_id_len; - - # Assuming that the client set up the connection, this can be considered the first - # received Initial from the client. So disable change of ConnectionID's afterwards - self.decrypted_data = decrypt_crypto_payload(self.fpack.data, - context.initial_destination_conn_id, - self.long_header.encrypted_offset, - self.long_header.payload_length, - from_client); - } - - # We attempted decryption, but it failed. Just reject the - # input and assume Zeek will disable the analyzer for this - # connection. - if ( |self.decrypted_data| == 0 ) - throw "decryption failed"; - } + # Slurp in the whole packet if we determined we have a chance to decrypt. + all_data: bytes &parse-at=self.start &eod if ( self?.long_header && can_decrypt(self.long_header, context, from_client) ) { + self.crypto_buffer = new CryptoBuffer(); + self.crypto_sink.connect(self.crypto_buffer); + + if ( from_client ) { + context.server_cid_len = self.long_header.dest_conn_id_len; + context.client_cid_len = self.long_header.src_conn_id_len; + + # This means that here, we can try to decrypt the initial packet! + # All data is accessible via the `long_header` unit + self.decrypted_data = decrypt_crypto_payload( + self.all_data, + self.long_header.dest_conn_id, + self.long_header.encrypted_offset, + self.long_header.payload_length, + from_client + ); - # If it's a reply from the server and it's not a REPLY, we assume the keys are restablished and decryption is no longer possible - # TODO: verify if this is actually correct per RFC - if (self.long_header.first_byte.packet_type != LongPacketType::RETRY && ! from_client) { - context.server_initial_processed = True; - context.client_initial_processed = True; - } + # Set this to be the seed for the decryption + if ( |context.initial_destination_conn_id| == 0 ) { + context.initial_destination_conn_id = self.long_header.dest_conn_id; + } + + } else { + context.server_cid_len = self.long_header.src_conn_id_len; + context.client_cid_len = self.long_header.dest_conn_id_len; + + # Assuming that the client set up the connection, this can be considered the first + # received Initial from the client. So disable change of ConnectionID's afterwards + self.decrypted_data = decrypt_crypto_payload( + self.all_data, + context.initial_destination_conn_id, + self.long_header.encrypted_offset, + self.long_header.payload_length, + from_client + ); } - }; + + # We attempted decryption, but it failed. Just reject the + # input and assume Zeek will disable the analyzer for this + # connection. + if ( |self.decrypted_data| == 0 ) + throw "decryption failed"; + + # If this was a reply from the server and it's not a RETRY, we assume the keys + # are restablished and decryption is no longer possible + # + # TODO: verify if this is actually correct per RFC + if ( self.long_header.first_byte.packet_type != LongPacketType::RETRY && ! from_client ) { + context.server_initial_processed = True; + context.client_initial_processed = True; + } + } # Depending on the type of header and whether we were able to decrypt # some of it, parse the remaining payload. - : ShortPacketPayload if (self.header_form == HeaderForm::SHORT); - : EncryptedLongPacketPayload if (self.header_form == HeaderForm::LONG && |self.decrypted_data| == 0); + : ShortPacketPayload if (self.first_byte.header_form == HeaderForm::SHORT); + : EncryptedLongPacketPayload if (self.first_byte.header_form == HeaderForm::LONG && |self.decrypted_data| == 0); # If this was packet with a long header and decrypted data exists, attempt # to parse the plain QUIC frames from it. - frames: Frame(self.long_header, from_client, self.crypto_sink)[] &parse-from=self.decrypted_data if (self.header_form == HeaderForm::LONG && |self.decrypted_data| > 0); + frames: Frame(self.long_header, from_client, self.crypto_sink)[] &parse-from=self.decrypted_data if (self.first_byte.header_form == HeaderForm::LONG && |self.decrypted_data| > 0); # Once the Packet is fully parsed, pass the accumulated CRYPTO frames # to the SSL analyzer as handshake data. on %done { - # print "packet done", zeek::is_orig(), self.header_form, |self.decrypted_data|; + # print "packet done", zeek::is_orig(), self.first_byte.header_form, |self.decrypted_data|; if ( self.crypto_buffer != Null && |self.crypto_buffer.buffered| > 0 ) { local handshake_data = self.crypto_buffer.buffered; diff --git a/analyzer/decrypt_crypto.cc b/analyzer/decrypt_crypto.cc index c2fcb49..7f7a02b 100644 --- a/analyzer/decrypt_crypto.cc +++ b/analyzer/decrypt_crypto.cc @@ -9,6 +9,7 @@ refactors as C++ development is not our main profession. */ // Default imports +#include #include #include #include @@ -23,16 +24,26 @@ refactors as C++ development is not our main profession. // Import HILTI #include +namespace + { + // Struct to store decryption info for this specific connection struct DecryptionInformation { std::vector unprotected_header; - std::vector protected_header; uint64_t packet_number; std::vector nonce; uint8_t packet_number_length; }; +// Return rt::hilti::Bytes::data() value as const uint8_t* +// +// This should be alright: https://stackoverflow.com/a/15172304 +inline const uint8_t* data_as_uint8(const hilti::rt::Bytes& b) + { + return reinterpret_cast(b.data()); + } + /* Constants used in the HKDF functions. HKDF-Expand-Label uses labels such as 'quic key' and 'quic hp'. These labels can obviously be @@ -75,7 +86,7 @@ const size_t MAXIMUM_PACKET_NUMBER_LENGTH = 4; /* HKDF-Extract as described in https://www.rfc-editor.org/rfc/rfc8446.html#section-7.1 */ -std::vector hkdf_extract(std::vector connection_id) +std::vector hkdf_extract(const hilti::rt::Bytes& connection_id) { std::vector out_temp(INITIAL_SECRET_LEN); size_t initial_secret_len = out_temp.size(); @@ -84,9 +95,9 @@ std::vector hkdf_extract(std::vector connection_id) EVP_PKEY_derive_init(pctx); EVP_PKEY_CTX_hkdf_mode(pctx, EVP_PKEY_HKDEF_MODE_EXTRACT_ONLY); EVP_PKEY_CTX_set_hkdf_md(pctx, digest); - EVP_PKEY_CTX_set1_hkdf_key(pctx, connection_id.data(), connection_id.size()); + EVP_PKEY_CTX_set1_hkdf_key(pctx, data_as_uint8(connection_id), connection_id.size()); EVP_PKEY_CTX_set1_hkdf_salt(pctx, INITIAL_SALT_V1.data(), INITIAL_SALT_V1.size()); - EVP_PKEY_derive(pctx, out_temp.data(), reinterpret_cast(&initial_secret_len)); + EVP_PKEY_derive(pctx, out_temp.data(), &initial_secret_len); EVP_PKEY_CTX_free(pctx); return out_temp; } @@ -95,8 +106,8 @@ std::vector hkdf_extract(std::vector connection_id) HKDF-Expand-Label as described in https://www.rfc-editor.org/rfc/rfc8446.html#section-7.1 that uses the global constant labels such as 'quic hp'. */ -std::vector hkdf_expand(size_t out_len, std::vector key, - std::vector info) +std::vector hkdf_expand(size_t out_len, const std::vector& key, + const std::vector& info) { std::vector out_temp(out_len); const EVP_MD* digest = EVP_sha256(); @@ -115,9 +126,9 @@ std::vector hkdf_expand(size_t out_len, std::vector key, Removes the header protection from the INITIAL packet and returns a DecryptionInformation struct that is partially filled */ -DecryptionInformation remove_header_protection(std::vector client_hp, +DecryptionInformation remove_header_protection(const std::vector& client_hp, uint64_t encrypted_offset, - std::vector encrypted_packet) + const hilti::rt::Bytes& all_data) { DecryptionInformation decryptInfo; int outlen; @@ -128,18 +139,19 @@ DecryptionInformation remove_header_protection(std::vector client_hp, // Passing an 1 means ENCRYPT EVP_CipherInit_ex(ctx, NULL, NULL, client_hp.data(), NULL, 1); - std::vector sample(encrypted_packet.begin() + encrypted_offset + - MAXIMUM_PACKET_NUMBER_LENGTH, + static_assert(AEAD_SAMPLE_LENGTH > 0); + assert(all_data.size() >= encrypted_offset + MAXIMUM_PACKET_NUMBER_LENGTH + AEAD_SAMPLE_LENGTH); + + const uint8_t* sample = data_as_uint8(all_data) + encrypted_offset + + MAXIMUM_PACKET_NUMBER_LENGTH; - encrypted_packet.begin() + encrypted_offset + - MAXIMUM_PACKET_NUMBER_LENGTH + AEAD_SAMPLE_LENGTH); - std::vector mask(sample.size()); - EVP_CipherUpdate(ctx, mask.data(), &outlen, sample.data(), AEAD_SAMPLE_LENGTH); + std::array mask; + EVP_CipherUpdate(ctx, mask.data(), &outlen, sample, AEAD_SAMPLE_LENGTH); EVP_CIPHER_CTX_free(ctx); // To determine the actual packet number length, // we have to remove the mask from the first byte - uint8_t first_byte = encrypted_packet[0]; + uint8_t first_byte = data_as_uint8(all_data)[0]; if ( first_byte & 0x80 ) { @@ -154,9 +166,8 @@ DecryptionInformation remove_header_protection(std::vector client_hp, int recovered_packet_number_length = (first_byte & 0x03) + 1; // .. and use this to reconstruct the (partially) unprotected header - std::vector unprotected_header(encrypted_packet.begin(), - - encrypted_packet.begin() + encrypted_offset + + std::vector unprotected_header(data_as_uint8(all_data), + data_as_uint8(all_data) + encrypted_offset + recovered_packet_number_length); uint32_t decoded_packet_number = 0; @@ -168,15 +179,11 @@ DecryptionInformation remove_header_protection(std::vector client_hp, decoded_packet_number = unprotected_header[encrypted_offset + i] | (decoded_packet_number << 8); } - std::vector protected_header(encrypted_packet.begin(), - encrypted_packet.begin() + encrypted_offset + - recovered_packet_number_length); // Store the information back in the struct decryptInfo.packet_number = decoded_packet_number; decryptInfo.packet_number_length = recovered_packet_number_length; - decryptInfo.protected_header = protected_header; - decryptInfo.unprotected_header = unprotected_header; + decryptInfo.unprotected_header = std::move(unprotected_header); return decryptInfo; } @@ -186,39 +193,48 @@ decoded packet number, and returns the nonce */ std::vector calculate_nonce(std::vector client_iv, uint64_t packet_number) { - std::vector nonce = client_iv; - for ( int i = 0; i < 8; ++i ) - { - nonce[AEAD_IV_LEN - 1 - i] ^= (uint8_t)(packet_number >> 8 * i); - } + client_iv[AEAD_IV_LEN - 1 - i] ^= (uint8_t)(packet_number >> 8 * i); - // Return the nonce - return nonce; + return client_iv; } /* Function that calls the AEAD decryption routine, and returns the decrypted data */ -std::vector decrypt(std::vector client_key, std::vector encrypted_packet, - uint64_t payload_length, DecryptionInformation decryptInfo) + +hilti::rt::Bytes decrypt(const std::vector& client_key, const hilti::rt::Bytes& all_data, + uint64_t payload_length, const DecryptionInformation& decryptInfo) { int out, out2, res; - std::vector encrypted_payload( - encrypted_packet.begin() + decryptInfo.protected_header.size(), - encrypted_packet.begin() + decryptInfo.protected_header.size() + payload_length - - decryptInfo.packet_number_length - AEAD_TAG_LENGTH); + if ( payload_length < decryptInfo.packet_number_length + AEAD_TAG_LENGTH ) + throw hilti::rt::RuntimeError( + hilti::rt::fmt("payload too small %ld < %ld", payload_length, + decryptInfo.packet_number_length + AEAD_TAG_LENGTH)); + + const uint8_t* encrypted_payload = data_as_uint8(all_data) + + decryptInfo.unprotected_header.size(); - std::vector tag_to_check( - encrypted_packet.begin() + decryptInfo.protected_header.size() + payload_length - - decryptInfo.packet_number_length - AEAD_TAG_LENGTH, + int encrypted_payload_size = payload_length - decryptInfo.packet_number_length - + AEAD_TAG_LENGTH; - encrypted_packet.begin() + decryptInfo.protected_header.size() + payload_length - - decryptInfo.packet_number_length); + if ( encrypted_payload_size < 0 ) + throw hilti::rt::RuntimeError( + hilti::rt::fmt("encrypted_payload_size underflow %ld", encrypted_payload_size)); - unsigned char decrypt_buffer[MAXIMUM_PACKET_LENGTH]; + if ( all_data.size() < + decryptInfo.unprotected_header.size() + encrypted_payload_size + AEAD_TAG_LENGTH ) + throw hilti::rt::RuntimeError( + hilti::rt::fmt("all_data too short %ld < %ld", all_data.size(), + decryptInfo.unprotected_header.size() + encrypted_payload_size)); + + const void* tag_to_check = all_data.data() + decryptInfo.unprotected_header.size() + + encrypted_payload_size; + int tag_to_check_length = AEAD_TAG_LENGTH; + + std::array decrypt_buffer; // Setup context auto cipher = EVP_aes_128_gcm(); @@ -235,7 +251,8 @@ std::vector decrypt(std::vector client_key, std::vector(tag_to_check)); // Setting the second parameter to NULL will pass it as Associated Data EVP_CipherUpdate(ctx, NULL, &out, decryptInfo.unprotected_header.data(), @@ -243,16 +260,16 @@ std::vector decrypt(std::vector client_key, std::vector decrypted_data(decrypt_buffer, decrypt_buffer + out); - return decrypted_data; + // Copy the decrypted data from the decrypted buffer into a Bytes instance. + return hilti::rt::Bytes(decrypt_buffer.data(), decrypt_buffer.data() + out); + } + } /* @@ -260,34 +277,21 @@ Function that is called from Spicy. It's a wrapper around `process_data`; it stores all the passed data in a global struct and then calls `process_data`, which will eventually return the decrypted data and pass it back to Spicy. */ -hilti::rt::Bytes decrypt_crypto_payload(const hilti::rt::Bytes& entire_packet, - const hilti::rt::Bytes& connection_id, - const hilti::rt::integer::safe& encrypted_offset, - const hilti::rt::integer::safe& payload_length, - const hilti::rt::Bool& from_client) +hilti::rt::Bytes +QUIC_decrypt_crypto_payload(const hilti::rt::Bytes& all_data, const hilti::rt::Bytes& connection_id, + const hilti::rt::integer::safe& encrypted_offset, + const hilti::rt::integer::safe& payload_length, + const hilti::rt::Bool& from_client) { if ( payload_length < 20 ) throw hilti::rt::RuntimeError(hilti::rt::fmt("payload too small %ld < 20", payload_length)); - if ( entire_packet.size() < encrypted_offset + payload_length ) - throw hilti::rt::RuntimeError(hilti::rt::fmt( - "packet too small %ld < %ld", entire_packet.size(), encrypted_offset + payload_length)); + if ( (all_data.size() < encrypted_offset + payload_length) ) + throw hilti::rt::RuntimeError(hilti::rt::fmt("packet too small %ld %ld", all_data.size(), + encrypted_offset + payload_length)); - // Fill in the entire packet bytes - std::vector e_pkt; - for ( const auto& singlebyte : entire_packet ) - { - e_pkt.push_back(singlebyte); - } - - std::vector cnnid; - for ( const auto& singlebyte : connection_id ) - { - cnnid.push_back(singlebyte); - } - - std::vector initial_secret = hkdf_extract(cnnid); + std::vector initial_secret = hkdf_extract(connection_id); std::vector server_client_secret; if ( from_client ) @@ -303,14 +307,10 @@ hilti::rt::Bytes decrypt_crypto_payload(const hilti::rt::Bytes& entire_packet, std::vector iv = hkdf_expand(AEAD_IV_LEN, server_client_secret, IV_INFO); std::vector hp = hkdf_expand(AEAD_HP_LEN, server_client_secret, HP_INFO); - DecryptionInformation decryptInfo = remove_header_protection(hp, encrypted_offset, e_pkt); + DecryptionInformation decryptInfo = remove_header_protection(hp, encrypted_offset, all_data); // Calculate the correct nonce for the decryption decryptInfo.nonce = calculate_nonce(iv, decryptInfo.packet_number); - std::vector decrypted_data = decrypt(key, e_pkt, payload_length, decryptInfo); - - // Return it as hilti Bytes again - hilti::rt::Bytes decr(decrypted_data.begin(), decrypted_data.end()); - return decr; + return decrypt(key, all_data, payload_length, decryptInfo); }