From 2434831f23bd900a87b9d9d38b4eb4990f120d8f Mon Sep 17 00:00:00 2001
From: Owen Voorhees <owen@squibner.com>
Date: Tue, 1 Sep 2020 21:12:23 -0700
Subject: [PATCH 1/2] Move LLVM bitstream utilities from swift-driver to TSC

---
 Sources/TSCUtility/Bits.swift                 |  96 +++++
 Sources/TSCUtility/Bitstream.swift            | 369 ++++++++++++++++++
 .../TSCUtility/SerializedDiagnostics.swift    |   8 +
 Tests/TSCUtilityTests/Inputs/clang.dia        | Bin 0 -> 1412 bytes
 Tests/TSCUtilityTests/Inputs/serialized.dia   | Bin 0 -> 2124 bytes
 .../SerializedDiagnosticsTests.swift          |   8 +
 6 files changed, 481 insertions(+)
 create mode 100644 Sources/TSCUtility/Bits.swift
 create mode 100644 Sources/TSCUtility/Bitstream.swift
 create mode 100644 Sources/TSCUtility/SerializedDiagnostics.swift
 create mode 100644 Tests/TSCUtilityTests/Inputs/clang.dia
 create mode 100644 Tests/TSCUtilityTests/Inputs/serialized.dia
 create mode 100644 Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift

diff --git a/Sources/TSCUtility/Bits.swift b/Sources/TSCUtility/Bits.swift
new file mode 100644
index 00000000..0a9e7ba2
--- /dev/null
+++ b/Sources/TSCUtility/Bits.swift
@@ -0,0 +1,96 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+
+struct Bits: RandomAccessCollection {
+  var buffer: Data
+
+  var startIndex: Int { return 0 }
+  var endIndex: Int { return buffer.count * 8 }
+
+  subscript(index: Int) -> UInt8 {
+    let byte = buffer[index / 8]
+    return (byte >> UInt8(index % 8)) & 1
+  }
+
+  func readBits(atOffset offset: Int, count: Int) -> UInt64 {
+    precondition(count >= 0 && count <= 64)
+    precondition(offset >= 0)
+    precondition(offset &+ count >= offset)
+    precondition(offset &+ count <= self.endIndex)
+
+    return buffer.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in
+      let upperBound = offset &+ count
+      let topByteIndex = upperBound >> 3
+      var result: UInt64 = 0
+      if upperBound & 7 != 0 {
+        let mask: UInt8 = (1 << UInt8(upperBound & 7)) &- 1
+        result = UInt64(bytes[topByteIndex] & mask)
+      }
+      for i in ((offset >> 3)..<(upperBound >> 3)).reversed() {
+        result <<= 8
+        result |= UInt64(bytes[i])
+      }
+      if offset & 7 != 0 {
+        result >>= UInt64(offset & 7)
+      }
+      return result
+    }
+  }
+
+  struct Cursor {
+    enum Error: Swift.Error { case bufferOverflow }
+
+    let buffer: Bits
+    private var offset: Int = 0
+
+    init(buffer: Bits) {
+      self.buffer = buffer
+    }
+    
+    init(buffer: Data) {
+      self.init(buffer: Bits(buffer: buffer))
+    }
+
+    var isAtEnd: Bool {
+      return offset == buffer.count
+    }
+
+    func peek(_ count: Int) throws -> UInt64 {
+      if buffer.count - offset < count { throw Error.bufferOverflow }
+      return buffer.readBits(atOffset: offset, count: count)
+    }
+
+    mutating func read(_ count: Int) throws -> UInt64 {
+      defer { offset += count }
+      return try peek(count)
+    }
+
+    mutating func read(bytes count: Int) throws -> Data {
+      precondition(count >= 0)
+      precondition(offset & 0b111 == 0)
+      let newOffset = offset &+ (count << 3)
+      precondition(newOffset >= offset)
+      if newOffset > buffer.count { throw Error.bufferOverflow }
+      defer { offset = newOffset }
+      return buffer.buffer.dropFirst(offset >> 3).prefix((newOffset - offset) >> 3)
+    }
+
+    mutating func advance(toBitAlignment align: Int) throws {
+      precondition(align > 0)
+      precondition(offset &+ (align&-1) >= offset)
+      precondition(align & (align &- 1) == 0)
+      if offset % align == 0 { return }
+      offset = (offset &+ align) & ~(align &- 1)
+      if offset > buffer.count { throw Error.bufferOverflow }
+    }
+  }
+}
diff --git a/Sources/TSCUtility/Bitstream.swift b/Sources/TSCUtility/Bitstream.swift
new file mode 100644
index 00000000..8c1dbafb
--- /dev/null
+++ b/Sources/TSCUtility/Bitstream.swift
@@ -0,0 +1,369 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+
+enum BitcodeElement {
+  struct Block {
+    var id: UInt64
+    var elements: [BitcodeElement]
+  }
+
+  struct Record {
+    enum Payload {
+      case none
+      case array([UInt64])
+      case char6String(String)
+      case blob(Data)
+    }
+
+    var id: UInt64
+    var fields: [UInt64]
+    var payload: Payload
+  }
+
+  case block(Block)
+  case record(Record)
+}
+
+struct BlockInfo {
+  var name: String = ""
+  var recordNames: [UInt64: String] = [:]
+}
+
+extension Bitcode {
+  struct Signature: Equatable {
+    private var value: UInt32
+
+    init(value: UInt32) {
+      self.value = value
+    }
+
+    init(string: String) {
+      precondition(string.utf8.count == 4)
+      var result: UInt32 = 0
+      for byte in string.utf8.reversed() {
+        result <<= 8
+        result |= UInt32(byte)
+      }
+      self.value = result
+    }
+  }
+}
+
+struct Bitcode {
+  let signature: Signature
+  let elements: [BitcodeElement]
+  let blockInfo: [UInt64: BlockInfo]
+}
+
+private extension Bits.Cursor {
+  enum BitcodeError: Swift.Error {
+    case vbrOverflow
+  }
+
+  mutating func readVBR(_ width: Int) throws -> UInt64 {
+    precondition(width > 1)
+    let testBit = UInt64(1 << (width &- 1))
+    let mask = testBit &- 1
+
+    var result: UInt64 = 0
+    var offset: UInt64 = 0
+    var next: UInt64
+    repeat {
+      next = try self.read(width)
+      result |= (next & mask) << offset
+      offset += UInt64(width &- 1)
+      if offset > 64 { throw BitcodeError.vbrOverflow }
+    } while next & testBit != 0
+
+    return result
+  }
+}
+
+private struct BitstreamReader {
+  struct Abbrev {
+    enum Operand {
+      case literal(UInt64)
+      case fixed(Int)
+      case vbr(Int)
+      indirect case array(Operand)
+      case char6
+      case blob
+
+      var isPayload: Bool {
+        switch self {
+        case .array, .blob: return true
+        case .literal, .fixed, .vbr, .char6: return false
+        }
+      }
+    }
+
+    var operands: [Operand] = []
+  }
+
+  enum Error: Swift.Error {
+    case invalidAbbrev
+    case nestedBlockInBlockInfo
+    case missingSETBID
+    case invalidBlockInfoRecord(recordID: UInt64)
+    case abbrevWidthTooSmall(width: Int)
+    case noSuchAbbrev(blockID: UInt64, abbrevID: Int)
+    case missingEndBlock(blockID: UInt64)
+  }
+
+  var cursor: Bits.Cursor
+  var blockInfo: [UInt64: BlockInfo] = [:]
+  var globalAbbrevs: [UInt64: [Abbrev]] = [:]
+
+  init(buffer: Data) {
+    cursor = Bits.Cursor(buffer: buffer)
+  }
+
+  mutating func readAbbrevOp() throws -> Abbrev.Operand {
+    let isLiteralFlag = try cursor.read(1)
+    if isLiteralFlag == 1 {
+      return .literal(try cursor.readVBR(8))
+    }
+
+    switch try cursor.read(3) {
+    case 0:
+      throw Error.invalidAbbrev
+    case 1:
+      return .fixed(Int(try cursor.readVBR(5)))
+    case 2:
+      return .vbr(Int(try cursor.readVBR(5)))
+    case 3:
+      return .array(try readAbbrevOp())
+    case 4:
+      return .char6
+    case 5:
+      return .blob
+    case 6, 7:
+      throw Error.invalidAbbrev
+    default:
+      fatalError()
+    }
+  }
+
+  mutating func readAbbrev(numOps: Int) throws -> Abbrev {
+    guard numOps > 0 else { throw Error.invalidAbbrev }
+
+    var operands: [Abbrev.Operand] = []
+    for i in 0..<numOps {
+      operands.append(try readAbbrevOp())
+
+      if case .array = operands.last! {
+        guard i == numOps - 2 else { throw Error.invalidAbbrev }
+        break
+      } else if case .blob = operands.last! {
+        guard i == numOps - 1 else { throw Error.invalidAbbrev }
+      }
+    }
+
+    return Abbrev(operands: operands)
+  }
+
+  mutating func readSingleAbbreviatedRecordOperand(_ operand: Abbrev.Operand) throws -> UInt64 {
+    switch operand {
+    case .char6:
+      let value = try cursor.read(6)
+      switch value {
+      case 0...25:
+        return value + UInt64(("a" as UnicodeScalar).value)
+      case 26...51:
+        return value + UInt64(("A" as UnicodeScalar).value) - 26
+      case 52...61:
+        return value + UInt64(("0" as UnicodeScalar).value) - 52
+      case 62:
+        return UInt64(("." as UnicodeScalar).value)
+      case 63:
+        return UInt64(("_" as UnicodeScalar).value)
+      default:
+        fatalError()
+      }
+    case .literal(let value):
+      return value
+    case .fixed(let width):
+      return try cursor.read(width)
+    case .vbr(let width):
+      return try cursor.readVBR(width)
+    case .array, .blob:
+      fatalError()
+    }
+  }
+
+  mutating func readAbbreviatedRecord(_ abbrev: Abbrev) throws -> BitcodeElement.Record {
+    let code = try readSingleAbbreviatedRecordOperand(abbrev.operands.first!)
+
+    let lastOperand = abbrev.operands.last!
+    let lastRegularOperandIndex: Int = abbrev.operands.endIndex - (lastOperand.isPayload ? 1 : 0)
+
+    var fields = [UInt64]()
+    for op in abbrev.operands[1..<lastRegularOperandIndex] {
+      fields.append(try readSingleAbbreviatedRecordOperand(op))
+    }
+
+    let payload: BitcodeElement.Record.Payload
+    if !lastOperand.isPayload {
+      payload = .none
+    } else {
+      switch lastOperand {
+      case .array(let element):
+        let length = try cursor.readVBR(6)
+        var elements = [UInt64]()
+        for _ in 0..<length {
+          elements.append(try readSingleAbbreviatedRecordOperand(element))
+        }
+        if case .char6 = element {
+          payload = .char6String(String(String.UnicodeScalarView(elements.map { UnicodeScalar(UInt8($0)) })))
+        } else {
+          payload = .array(elements)
+        }
+      case .blob:
+        let length = Int(try cursor.readVBR(6))
+        try cursor.advance(toBitAlignment: 32)
+        payload = .blob(try cursor.read(bytes: length))
+        try cursor.advance(toBitAlignment: 32)
+      default:
+        fatalError()
+      }
+    }
+
+    return .init(id: code, fields: fields, payload: payload)
+  }
+
+  mutating func readBlockInfoBlock(abbrevWidth: Int) throws {
+    var currentBlockID: UInt64?
+    while true {
+      switch try cursor.read(abbrevWidth) {
+      case 0: // END_BLOCK
+        try cursor.advance(toBitAlignment: 32)
+        // FIXME: check expected length
+        return
+
+      case 1: // ENTER_BLOCK
+        throw Error.nestedBlockInBlockInfo
+
+      case 2: // DEFINE_ABBREV
+        guard let blockID = currentBlockID else {
+          throw Error.missingSETBID
+        }
+        let numOps = Int(try cursor.readVBR(5))
+        if globalAbbrevs[blockID] == nil { globalAbbrevs[blockID] = [] }
+        globalAbbrevs[blockID]!.append(try readAbbrev(numOps: numOps))
+
+      case 3: // UNABBREV_RECORD
+        let code = try cursor.readVBR(6)
+        let numOps = try cursor.readVBR(6)
+        var operands = [UInt64]()
+        for _ in 0..<numOps {
+          operands.append(try cursor.readVBR(6))
+        }
+
+        switch code {
+        case 1:
+          guard operands.count == 1 else { throw Error.invalidBlockInfoRecord(recordID: code) }
+          currentBlockID = operands.first
+        case 2:
+          guard let blockID = currentBlockID else {
+            throw Error.missingSETBID
+          }
+          if blockInfo[blockID] == nil { blockInfo[blockID] = BlockInfo() }
+          blockInfo[blockID]!.name = String(bytes: operands.map { UInt8($0) }, encoding: .utf8) ?? "<invalid>"
+        case 3:
+          guard let blockID = currentBlockID else {
+            throw Error.missingSETBID
+          }
+          if blockInfo[blockID] == nil { blockInfo[blockID] = BlockInfo() }
+          guard let recordID = operands.first else {
+            throw Error.invalidBlockInfoRecord(recordID: code)
+          }
+          blockInfo[blockID]!.recordNames[recordID] = String(bytes: operands.dropFirst().map { UInt8($0) }, encoding: .utf8) ?? "<invalid>"
+        default:
+          throw Error.invalidBlockInfoRecord(recordID: code)
+        }
+
+      case let abbrevID:
+        throw Error.noSuchAbbrev(blockID: 0, abbrevID: Int(abbrevID))
+      }
+    }
+  }
+
+  mutating func readBlock(id: UInt64, abbrevWidth: Int, abbrevInfo: [Abbrev]) throws -> [BitcodeElement] {
+    var abbrevInfo = abbrevInfo
+    var elements = [BitcodeElement]()
+
+    while !cursor.isAtEnd {
+      switch try cursor.read(abbrevWidth) {
+      case 0: // END_BLOCK
+        try cursor.advance(toBitAlignment: 32)
+        // FIXME: check expected length
+        return elements
+
+      case 1: // ENTER_SUBBLOCK
+        let blockID = try cursor.readVBR(8)
+        let newAbbrevWidth = Int(try cursor.readVBR(4))
+        try cursor.advance(toBitAlignment: 32)
+        _ = try cursor.read(32) // FIXME: use expected length
+
+        switch blockID {
+        case 0:
+          try readBlockInfoBlock(abbrevWidth: newAbbrevWidth)
+        case 1...7:
+          // Metadata blocks we don't understand yet
+          fallthrough
+        default:
+          let innerElements = try readBlock(
+            id: blockID, abbrevWidth: newAbbrevWidth, abbrevInfo: globalAbbrevs[blockID] ?? [])
+          elements.append(.block(.init(id: blockID, elements: innerElements)))
+        }
+
+      case 2: // DEFINE_ABBREV
+        let numOps = Int(try cursor.readVBR(5))
+        abbrevInfo.append(try readAbbrev(numOps: numOps))
+
+      case 3: // UNABBREV_RECORD
+        let code = try cursor.readVBR(6)
+        let numOps = try cursor.readVBR(6)
+        var operands = [UInt64]()
+        for _ in 0..<numOps {
+          operands.append(try cursor.readVBR(6))
+        }
+        elements.append(.record(.init(id: code, fields: operands, payload: .none)))
+
+      case let abbrevID:
+        guard Int(abbrevID) - 4 < abbrevInfo.count else {
+          throw Error.noSuchAbbrev(blockID: id, abbrevID: Int(abbrevID))
+        }
+        elements.append(.record(try readAbbreviatedRecord(abbrevInfo[Int(abbrevID) - 4])))
+      }
+    }
+
+    guard id == Self.fakeTopLevelBlockID else {
+      throw Error.missingEndBlock(blockID: id)
+    }
+    return elements
+  }
+
+  static let fakeTopLevelBlockID: UInt64 = ~0
+}
+
+extension Bitcode {
+  init(data: Data) throws {
+    precondition(data.count > 4)
+    let signatureValue = UInt32(Bits(buffer: data).readBits(atOffset: 0, count: 32))
+    let bitstreamData = data[4..<data.count]
+
+    var reader = BitstreamReader(buffer: bitstreamData)
+    let topLevelElements = try reader.readBlock(id: BitstreamReader.fakeTopLevelBlockID, abbrevWidth: 2, abbrevInfo: [])
+    self.init(signature: .init(value: signatureValue), elements: topLevelElements, blockInfo: reader.blockInfo)
+  }
+}
diff --git a/Sources/TSCUtility/SerializedDiagnostics.swift b/Sources/TSCUtility/SerializedDiagnostics.swift
new file mode 100644
index 00000000..25dad342
--- /dev/null
+++ b/Sources/TSCUtility/SerializedDiagnostics.swift
@@ -0,0 +1,8 @@
+//
+//  SerializedDiagnostics.swift
+//  swift-tools-support-core
+//
+//  Created by Owen Voorhees on 9/1/20.
+//
+
+import Foundation
diff --git a/Tests/TSCUtilityTests/Inputs/clang.dia b/Tests/TSCUtilityTests/Inputs/clang.dia
new file mode 100644
index 0000000000000000000000000000000000000000..97f32c952100b8ed19c5a5efff77634f1b723d69
GIT binary patch
literal 1412
zcmc&yO-vI(6n-rQY58eGY8Cl$AYvCHzZwF?==KLGnjl&s{vBM}p={jlmhQHo;ZRUS
zPZGm{i%CtIcrzS4aPUyW$%7g>c_1+)O5(v7V@y1F@J+Y1Az*?hw|RXt@0<B%zVD51
z&^=%%2j~ERGGoa2&hm-3Zmt;2(v#YcLjC$?-G<;!`pa#c!}Q8DXJ2oqeIs}q7d@`K
zYrW~YzU8$($$#(4;=<(Od^EbCatjsyPRk~5eP*ltGF(|xrEhs@f1lkrw(N1Oczahp
zJ<q+pQb%RLzi-B0eEYKP$i?E&{VL96ne{Ajv+jOJY5Clx&q;39ld^F>v)wq)H$38(
zGO4bL>@1VIYR#3;;tJg7HZ-6PBkI7E{hDk57jSeiHwWIqx8b;bM5QscU74bCLQ!H9
zl&bAMB^ZxTS;G-cP)Uih2^))&swuGu2?Uk+E`{@>K0y<_LU4k%MUeXk+z<?yM+GTP
ziC2*mc$BEfWKdP(6yN}u@CTqyC^3!7t=%VDHKp}<R~lBa+KsWbAE3T8W6od(5F=4Z
z42l{F7N{ger796+jfQEAL}N54vLjr?ITRSwt4_iRHwEI96gli{CYnN=xZVliz#BlO
z1mlGfc({&73}Dok<scp*{tt8rp>G!a1Loy5%p-=5PNwe+dLn2fpv6QvoD=6v8j=hf
z&td>+_=(dhjF};T78|J`bqGU(d7~0#oM^YAbla`bmbermvZ9eOO5!RF5kVz7ArWF>
zy+?gS2q7R7I-Pe%P_?YetYGv;)u<4{vm9_51vLUB-<nbHFDyO8IF+@6K?I@KdFT#8
zJ$r$+5kANfR%g%Jm9Pkk&_gK%`kGz0mjHUtF#*j;h#&L(DCRV5p@%LUZJc9oBUPHR
zGyLD)MqF0Ku#AT478RZZI5H=Z4PPZc&5ZbtlfxKao6tT6CGqZ=sXSyw=mSFf{f058
z;h0hHvuotP^*PzE_nCicK@AQxjP?0(3u<6crUoOGhb9r4-vh)RPJxj1nGsh1ov_g7
Lo%hLa-_hsaPPteK

literal 0
HcmV?d00001

diff --git a/Tests/TSCUtilityTests/Inputs/serialized.dia b/Tests/TSCUtilityTests/Inputs/serialized.dia
new file mode 100644
index 0000000000000000000000000000000000000000..51aebca3f8362f0b77b0c3cda3912df3a9034242
GIT binary patch
literal 2124
zcmcIl%}*0S6rWw&@C}s!q9PfHW{W|Jh65j$r7f`-zfpq51Jms*tW0<7?v|?2WFa8N
z6EPe(V&d6w;NY>wlNS>?dGLbKg9rZy-)u{PO*97UG}*qH%=^8M-+OQ3<D(a)dP4dM
zA$4*}{#5&2ZQ9wCYWS;;Z>)2B=hPQA>L%(#;kLksz+%gGSI0*d>t2lxFTEdny*RS|
zeuO8U-C13Eu)3VhuGry~hQ#^W9kuCAsPX6Z#*@uOm3J+lJ;mgDba*p1^fo%U6&vFH
zjT4EgCyC(WsnF@0!PN8SaG-V}x)xp-9c!zuU%d6h4KGADLg9E#i@dCMy;Ro<8y6b9
zU18&HQ|b5;-ax%ksSD$fF%JCm)|n6?=TK@d?Fl(fhER^&w!yL^W*&@r)3jy*>`2_y
zav3ljlwcmXnH8Hzl<USEO`8_I!ZhD*+<{CS9H#S09r7{La4eH^u=?!0o_4m#eY7-K
zwyXwJ&n<YE1<ipJ%|Ipz7ByG~&;aLVUN>f_(x((vgzs%A`f;YKrklhKB?W1g<Bmc(
zO9H1FGpK3sN0eo=z$~g8ww?mYsDkb|+N?rthoQ$9m1jWUANBCR;#Xv?xC(*7K|htJ
z3S|xjBSp<LO<GI?9~2Q-b*R2Y4q~~oE0^O<fySA|9Njd8rCn~qMYDvs?XJjHWw@De
zObaG3587-D8BXa2!;>O2gbYY$(b9`u%j7WJw4$~jryv*h;#GPv<&YI2n-(EGYLF0B
zWE_`Eg!-5#6C&Y{^p-XvFU=_XvDKj3Ag+}2iGl^Tjp;<LDc5aB@UwSn7Xby{|May7
z&FocqtJiOp;V@n-aJ$q`(&Yn|4#Sz(Pd=RKO81krgnm}2pL|$<(@%*a?O1Ea)CkIz
z;o>SddDEI3&#)PYF>MxhTZs(FooMCWu2n*U<Y1lR6fzt=t>dg?X$QYJ;`WuPP%PJ|
z59?3;G+sD@eI%xiSdy5DeV+Kce)3`eb3gg;D&?yPVyYB~m;_={9hSpuNqpz#S(2CN
z+LGglcA|{#23?P<oDE`j<P01rCZEU0X&-4yT(`3fUyvI)$bmBHR%2~p?!{82Zv2)3
Fe*q>}`2zp|

literal 0
HcmV?d00001

diff --git a/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift b/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift
new file mode 100644
index 00000000..f5fc7bda
--- /dev/null
+++ b/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift
@@ -0,0 +1,8 @@
+//
+//  File.swift
+//  
+//
+//  Created by Owen Voorhees on 9/1/20.
+//
+
+import Foundation

From 08bd94f11a6540ceb97bba4073bca1ec43300a03 Mon Sep 17 00:00:00 2001
From: Owen Voorhees <owen@squibner.com>
Date: Tue, 1 Sep 2020 21:12:56 -0700
Subject: [PATCH 2/2] Add deserialization for the Swift/Clang serialized
 diagnostics (.dia) format

---
 Sources/TSCUtility/CMakeLists.txt             |   3 +
 .../TSCUtility/SerializedDiagnostics.swift    | 236 +++++++++++++++++-
 .../SerializedDiagnosticsTests.swift          |  89 ++++++-
 3 files changed, 316 insertions(+), 12 deletions(-)

diff --git a/Sources/TSCUtility/CMakeLists.txt b/Sources/TSCUtility/CMakeLists.txt
index c0492d11..19edf343 100644
--- a/Sources/TSCUtility/CMakeLists.txt
+++ b/Sources/TSCUtility/CMakeLists.txt
@@ -10,6 +10,8 @@ add_library(TSCUtility
   Archiver.swift
   ArgumentParser.swift
   ArgumentParserShellCompletion.swift
+  Bits.swift
+  Bitstream.swift
   BuildFlags.swift
   CollectionExtensions.swift
   Diagnostics.swift
@@ -28,6 +30,7 @@ add_library(TSCUtility
   Platform.swift
   PolymorphicCodable.swift
   ProgressAnimation.swift
+  SerializedDiagnostics.swift
   SQLite.swift
   SimplePersistence.swift
   StringExtensions.swift
diff --git a/Sources/TSCUtility/SerializedDiagnostics.swift b/Sources/TSCUtility/SerializedDiagnostics.swift
index 25dad342..674850c5 100644
--- a/Sources/TSCUtility/SerializedDiagnostics.swift
+++ b/Sources/TSCUtility/SerializedDiagnostics.swift
@@ -1,8 +1,232 @@
-//
-//  SerializedDiagnostics.swift
-//  swift-tools-support-core
-//
-//  Created by Owen Voorhees on 9/1/20.
-//
+/*
+ This source file is part of the Swift.org open source project
 
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
 import Foundation
+
+/// Represents diagnostics serialized in a .dia file by the Swift compiler or Clang.
+public struct SerializedDiagnostics {
+  public enum Error: Swift.Error {
+    case badMagic
+    case unexpectedTopLevelRecord
+    case unknownBlock
+    case malformedRecord
+    case noMetadataBlock
+    case unexpectedSubblock
+    case unexpectedRecord
+    case missingInformation
+  }
+
+  private enum BlockID: UInt64 {
+    case metadata = 8
+    case diagnostic = 9
+  }
+
+  private enum RecordID: UInt64 {
+    case version = 1
+    case diagnosticInfo = 2
+    case sourceRange = 3
+    case flag = 4
+    case category = 5
+    case filename = 6
+    case fixit = 7
+  }
+
+  /// The serialized diagnostics format version number.
+  public var versionNumber: Int
+  /// Serialized diagnostics.
+  public var diagnostics: [Diagnostic]
+
+  public init(data: Data) throws {
+    let bitcode = try Bitcode(data: data)
+
+    guard bitcode.signature == .init(string: "DIAG") else { throw Error.badMagic }
+
+    var diagnostics: [Diagnostic] = []
+    var versionNumber: Int? = nil
+    var filenameMap = [UInt64: String]()
+    var flagMap = [UInt64: String]()
+    var categoryMap = [UInt64: String]()
+
+    for element in bitcode.elements {
+      guard case let .block(block) = element else { throw Error.unexpectedTopLevelRecord }
+      switch BlockID(rawValue: block.id) {
+      case .metadata:
+        guard block.elements.count == 1,
+              case let .record(versionRecord) = block.elements[0],
+              versionRecord.id == RecordID.version.rawValue,
+              versionRecord.fields.count == 1 else {
+          throw Error.malformedRecord
+        }
+        versionNumber = Int(versionRecord.fields[0])
+      case .diagnostic:
+        diagnostics.append(try Diagnostic(block: block,
+                                          filenameMap: &filenameMap,
+                                          flagMap: &flagMap,
+                                          categoryMap: &categoryMap))
+      case nil:
+        throw Error.unknownBlock
+      }
+    }
+
+    guard let version = versionNumber else { throw Error.noMetadataBlock }
+    self.versionNumber = version
+    self.diagnostics = diagnostics
+  }
+}
+
+extension SerializedDiagnostics {
+  public struct Diagnostic {
+
+    public enum Level: UInt64 {
+      case ignored, note, warning, error, fatal, remark
+    }
+    /// The diagnostic message text.
+    public var text: String
+    /// The level the diagnostic was emitted at.
+    public var level: Level
+    /// The location the diagnostic was emitted at in the source file.
+    public var location: SourceLocation
+    /// The diagnostic category. Currently only Clang emits this.
+    public var category: String?
+    /// The corresponding diagnostic command-line flag. Currently only Clang emits this.
+    public var flag: String?
+    /// Ranges in the source file associated with the diagnostic.
+    public var ranges: [(SourceLocation, SourceLocation)]
+    /// Fix-its associated with the diagnostic.
+    public var fixIts: [FixIt]
+
+    fileprivate init(block: BitcodeElement.Block,
+                     filenameMap: inout [UInt64: String],
+                     flagMap: inout [UInt64: String],
+                     categoryMap: inout [UInt64: String]) throws {
+      var text: String? = nil
+      var level: Level? = nil
+      var location: SourceLocation? = nil
+      var category: String? = nil
+      var flag: String? = nil
+      var ranges: [(SourceLocation, SourceLocation)] = []
+      var fixIts: [FixIt] = []
+
+      for element in block.elements {
+        guard case let .record(record) = element else {
+          throw Error.unexpectedSubblock
+        }
+
+        switch SerializedDiagnostics.RecordID(rawValue: record.id) {
+        case .diagnosticInfo:
+          guard record.fields.count == 8,
+                case .blob(let diagnosticBlob) = record.payload
+          else { throw Error.malformedRecord }
+
+          text = String(data: diagnosticBlob, encoding: .utf8)
+          level = Level(rawValue: record.fields[0])
+          location = try SourceLocation(fields: record.fields[1...4],
+                                        filenameMap: filenameMap)
+          category = categoryMap[record.fields[5]]
+          flag = flagMap[record.fields[6]]
+
+        case .sourceRange:
+          guard record.fields.count == 8 else { throw Error.malformedRecord }
+
+          let start = try SourceLocation(fields: record.fields[0...3],
+                                         filenameMap: filenameMap)
+          let end = try SourceLocation(fields: record.fields[4...7],
+                                       filenameMap: filenameMap)
+          ranges.append((start, end))
+
+        case .flag:
+          guard record.fields.count == 2,
+                case .blob(let flagBlob) = record.payload,
+                let flagText = String(data: flagBlob, encoding: .utf8)
+          else { throw Error.malformedRecord }
+
+          let diagnosticID = record.fields[0]
+          flagMap[diagnosticID] = flagText
+
+        case .category:
+          guard record.fields.count == 2,
+                case .blob(let categoryBlob) = record.payload,
+                let categoryText = String(data: categoryBlob, encoding: .utf8)
+          else { throw Error.malformedRecord }
+
+          let categoryID = record.fields[0]
+          categoryMap[categoryID] = categoryText
+
+        case .filename:
+          guard record.fields.count == 4,
+                case .blob(let filenameBlob) = record.payload,
+                let filenameText = String(data: filenameBlob, encoding: .utf8)
+          else { throw Error.malformedRecord }
+
+          let filenameID = record.fields[0]
+          // record.fields[1] and record.fields[2] are no longer used.
+          filenameMap[filenameID] = filenameText
+
+        case .fixit:
+          guard record.fields.count == 9,
+                case .blob(let fixItBlob) = record.payload,
+                let fixItText = String(data: fixItBlob, encoding: .utf8)
+          else { throw Error.malformedRecord }
+
+          let start = try SourceLocation(fields: record.fields[0...3],
+                                         filenameMap: filenameMap)
+          let end = try SourceLocation(fields: record.fields[4...7],
+                                       filenameMap: filenameMap)
+          fixIts.append(FixIt(start: start, end: end, text: fixItText))
+
+        case .version, nil:
+          throw Error.unexpectedRecord
+        }
+      }
+
+      do {
+        guard let text = text, let level = level, let location = location else {
+          throw Error.missingInformation
+        }
+        self.text = text
+        self.level = level
+        self.location = location
+        self.category = category
+        self.flag = flag
+        self.fixIts = fixIts
+        self.ranges = ranges
+      }
+    }
+  }
+
+  public struct SourceLocation: Equatable {
+    /// The filename associated with the diagnostic.
+    public var filename: String
+    public var line: UInt64
+    public var column: UInt64
+    /// The byte offset in the source file of the diagnostic. Currently, only
+    /// Clang includes this, it is set to 0 by Swift.
+    public var offset: UInt64
+
+    fileprivate init(fields: ArraySlice<UInt64>,
+                     filenameMap: [UInt64: String]) throws {
+      guard let name = filenameMap[fields[fields.startIndex]] else {
+        throw Error.missingInformation
+      }
+      self.filename = name
+      self.line = fields[fields.startIndex + 1]
+      self.column = fields[fields.startIndex + 2]
+      self.offset = fields[fields.startIndex + 3]
+    }
+  }
+
+  public struct FixIt {
+    /// Start location.
+    public var start: SourceLocation
+    /// End location.
+    public var end: SourceLocation
+    /// Fix-it replacement text.
+    public var text: String
+  }
+}
diff --git a/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift b/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift
index f5fc7bda..1d22d485 100644
--- a/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift
+++ b/Tests/TSCUtilityTests/SerializedDiagnosticsTests.swift
@@ -1,8 +1,85 @@
-//
-//  File.swift
-//  
-//
-//  Created by Owen Voorhees on 9/1/20.
-//
+/*
+ This source file is part of the Swift.org open source project
 
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import XCTest
 import Foundation
+import TSCBasic
+import TSCUtility
+
+final class SerializedDiagnosticsTests: XCTestCase {
+  func testReadSwiftSerializedDiags() throws {
+    let serializedDiagnosticsPath = AbsolutePath(#file).parentDirectory
+        .appending(components: "Inputs", "serialized.dia")
+    let contents = try localFileSystem.readFileContents(serializedDiagnosticsPath)
+    let serializedDiags = try SerializedDiagnostics(data: Data(contents.contents))
+
+    XCTAssertEqual(serializedDiags.versionNumber, 1)
+    XCTAssertEqual(serializedDiags.diagnostics.count, 17)
+
+    let one = serializedDiags.diagnostics[5]
+    XCTAssertEqual(one.text, "expected ',' separator")
+    XCTAssertEqual(one.level, .error)
+    XCTAssertTrue(one.location.filename.hasSuffix("/StoreSearchCoordinator.swift"))
+    XCTAssertEqual(one.location.line, 21)
+    XCTAssertEqual(one.location.column, 69)
+    XCTAssertEqual(one.location.offset, 0)
+    XCTAssertNil(one.category)
+    XCTAssertNil(one.flag)
+    XCTAssertEqual(one.ranges.count, 0)
+    XCTAssertEqual(one.fixIts.count, 1)
+    XCTAssertEqual(one.fixIts[0].text, ",")
+    XCTAssertEqual(one.fixIts[0].start, one.fixIts[0].end)
+    XCTAssertTrue(one.fixIts[0].start.filename.hasSuffix("/StoreSearchCoordinator.swift"))
+    XCTAssertEqual(one.fixIts[0].start.line, 21)
+    XCTAssertEqual(one.fixIts[0].start.column, 69)
+    XCTAssertEqual(one.fixIts[0].start.offset, 0)
+
+    let two = serializedDiags.diagnostics[16]
+    XCTAssertEqual(two.text, "use of unresolved identifier 'DispatchQueue'")
+    XCTAssertEqual(two.level, .error)
+    XCTAssertTrue(two.location.filename.hasSuffix("/Observable.swift"))
+    XCTAssertEqual(two.location.line, 34)
+    XCTAssertEqual(two.location.column, 13)
+    XCTAssertEqual(two.location.offset, 0)
+    XCTAssertNil(two.category)
+    XCTAssertNil(two.flag)
+    XCTAssertEqual(two.ranges.count, 1)
+    XCTAssertTrue(two.ranges[0].0.filename.hasSuffix("/Observable.swift"))
+    XCTAssertEqual(two.ranges[0].0.line, 34)
+    XCTAssertEqual(two.ranges[0].0.column, 13)
+    XCTAssertEqual(two.ranges[0].0.offset, 0)
+    XCTAssertTrue(two.ranges[0].1.filename.hasSuffix("/Observable.swift"))
+    XCTAssertEqual(two.ranges[0].1.line, 34)
+    XCTAssertEqual(two.ranges[0].1.column, 26)
+    XCTAssertEqual(two.ranges[0].1.offset, 0)
+    XCTAssertEqual(two.fixIts.count, 0)
+  }
+
+  func testReadClangSerializedDiags() throws {
+    let serializedDiagnosticsPath = AbsolutePath(#file).parentDirectory
+        .appending(components: "Inputs", "clang.dia")
+    let contents = try localFileSystem.readFileContents(serializedDiagnosticsPath)
+    let serializedDiags = try SerializedDiagnostics(data: Data(contents.contents))
+
+    XCTAssertEqual(serializedDiags.versionNumber, 1)
+    XCTAssertEqual(serializedDiags.diagnostics.count, 4)
+
+    let one = serializedDiags.diagnostics[1]
+    XCTAssertEqual(one.text, "values of type 'NSInteger' should not be used as format arguments; add an explicit cast to 'long' instead")
+    XCTAssertEqual(one.level, .warning)
+    XCTAssertEqual(one.location.line, 252)
+    XCTAssertEqual(one.location.column, 137)
+    XCTAssertEqual(one.location.offset, 10046)
+    XCTAssertEqual(one.category, "Format String Issue")
+    XCTAssertEqual(one.flag, "format")
+    XCTAssertEqual(one.ranges.count, 4)
+    XCTAssertEqual(one.fixIts.count, 2)
+  }
+}