hashfile: change default local hash to md5raw

src/dvc_data/hashfile/build.py

@@ -257,7 +257,7 @@ def build(
             **kwargs,
         )
         logger.debug("built tree '%s'", obj)
-        if name != "md5":
+        if name not in ("md5", "md5raw"):
             obj = _build_external_tree_info(odb, obj, name)
     else:
         meta, obj = _build_file(

src/dvc_data/hashfile/hash.py

@@ -23,14 +23,20 @@ def dos2unix(data: bytes) -> bytes:
     return data.replace(b"\r\n", b"\n")
 
 
-algorithms_available = hashlib.algorithms_available | {"blake3"}
+# NOTE: For legacy reasons "md5" is equivalent to DVC dos2unix MD5. For raw
+# unmodified MD5 use "md5raw".
+algorithms_available = hashlib.algorithms_available | {"blake3", "md5raw"}
+DEFAULT_ALGORITHM = "md5raw"
+LEGACY_HASH_NAMES = {"md5", "etag", "checksum"}
 
 
 def get_hasher(name: str) -> "hashlib._Hash":
     if name == "blake3":
         from blake3 import blake3
 
         return blake3(max_threads=blake3.AUTO)
+    if name == "md5raw":
+        name = "md5"
 
     try:
         return getattr(hashlib, name)()
@@ -42,13 +48,12 @@ class HashStreamFile(io.IOBase):
     def __init__(
         self,
         fobj: BinaryIO,
-        hash_name: str = "md5",
-        text: Optional[bool] = None,
+        hash_name: str = DEFAULT_ALGORITHM,
     ) -> None:
         self.fobj = fobj
         self.total_read = 0
+        hash_name = hash_name.lower()
         self.hasher = get_hasher(hash_name)
-        self.is_text: Optional[bool] = text
         super().__init__()
 
     def readable(self) -> bool:
@@ -59,13 +64,8 @@ def tell(self) -> int:
 
     def read(self, n=-1) -> bytes:
         chunk = self.fobj.read(n)
-        if self.is_text is None and chunk:
-            # do we need to buffer till the DEFAULT_CHUNK_SIZE?
-            self.is_text = istextblock(chunk[:DEFAULT_CHUNK_SIZE])
-
-        data = dos2unix(chunk) if self.is_text else chunk
-        self.hasher.update(data)
-        self.total_read += len(data)
+        self.hasher.update(chunk)
+        self.total_read += len(chunk)
         return chunk
 
     @property
@@ -77,17 +77,35 @@ def hash_name(self) -> str:
         return self.hasher.name
 
 
+class Dos2UnixHashStreamFile(HashStreamFile):
+    def read(self, n=-1) -> bytes:
+        # ideally, we want the heuristics to be applied in a similar way,
+        # regardless of the size of the first chunk,
+        # for which we may need to buffer till DEFAULT_CHUNK_SIZE.
+        assert n >= DEFAULT_CHUNK_SIZE
+        chunk = self.fobj.read(n)
+        is_text = istextblock(chunk[:DEFAULT_CHUNK_SIZE]) if chunk else False
+
+        data = dos2unix(chunk) if is_text else chunk
+        self.hasher.update(data)
+        self.total_read += len(data)
+        return chunk
+
+
+def get_hash_stream(
+    fobj: BinaryIO, name: str = DEFAULT_ALGORITHM
+) -> HashStreamFile:
+    cls = Dos2UnixHashStreamFile if name == "md5" else HashStreamFile
+    return cls(fobj, hash_name=name)
+
+
 def fobj_md5(
     fobj: BinaryIO,
     chunk_size: int = 2**20,
     text: Optional[bool] = None,
-    name="md5",
+    name: str = DEFAULT_ALGORITHM,
 ) -> str:
-    # ideally, we want the heuristics to be applied in a similar way,
-    # regardless of the size of the first chunk,
-    # for which we may need to buffer till DEFAULT_CHUNK_SIZE.
-    assert chunk_size >= DEFAULT_CHUNK_SIZE
-    stream = HashStreamFile(fobj, hash_name=name, text=text)
+    stream = get_hash_stream(fobj, name=name)
     while True:
         data = stream.read(chunk_size)
         if not data:
@@ -100,7 +118,7 @@ def file_md5(
     fs: "FileSystem" = localfs,
     callback: "Callback" = DEFAULT_CALLBACK,
     text: Optional[bool] = None,
-    name: str = "md5",
+    name: str = DEFAULT_ALGORITHM,
     size: Optional[int] = None,
 ) -> str:
     if size is None:
@@ -130,8 +148,11 @@ def _hash_file(
         func = getattr(fs, name)
         return str(func(path)), meta
 
-    if name == "md5":
-        return file_md5(path, fs, callback=callback, size=meta.size), meta
+    if name in ("md5", "md5raw"):
+        return (
+            file_md5(path, fs, callback=callback, size=meta.size, name=name),
+            meta,
+        )
     raise NotImplementedError
 
 

src/dvc_data/hashfile/meta.py

@@ -14,6 +14,7 @@ class Meta:
     PARAM_ETAG: ClassVar[str] = "etag"
     PARAM_CHECKSUM: ClassVar[str] = "checksum"
     PARAM_MD5: ClassVar[str] = "md5"
+    PARAM_MD5RAW: ClassVar[str] = "md5raw"
     PARAM_INODE: ClassVar[str] = "inode"
     PARAM_MTIME: ClassVar[str] = "mtime"
     PARAM_REMOTE: ClassVar[str] = "remote"
@@ -28,6 +29,7 @@ class Meta:
     etag: Optional[str] = None
     checksum: Optional[str] = None
     md5: Optional[str] = None
+    md5raw: Optional[str] = None
     inode: Optional[int] = None
     mtime: Optional[float] = None
 
@@ -67,6 +69,7 @@ def from_info(
             etag=etag,
             checksum=checksum,
             md5=info.get("md5"),
+            md5raw=info.get("md5raw"),
             inode=info.get("ino"),
             mtime=info.get("mtime"),
             remote=info.get("remote"),
@@ -107,6 +110,9 @@ def to_dict(self) -> Dict[str, Any]:
         if self.md5:
             ret[self.PARAM_MD5] = self.md5
 
+        if self.md5raw:
+            ret[self.PARAM_MD5RAW] = self.md5raw
+
         if self.remote:
             ret[self.PARAM_REMOTE] = self.remote