From b535937fca1054cc7e72ba37519cf5c7ca23a1f7 Mon Sep 17 00:00:00 2001 From: Gautham <41098605+ahgamut@users.noreply.github.com> Date: Sun, 26 Jun 2022 18:21:00 +0530 Subject: [PATCH] Add sys.meta_path entry for APE zip store (#425) --- third_party/python/Include/import.h | 2 + .../python/Lib/importlib/_bootstrap.py | 33 +- .../Lib/importlib/_bootstrap_external.py | 4 +- third_party/python/Lib/io.py | 6 +- third_party/python/Lib/ntpath.py | 15 +- third_party/python/Lib/test/test_cmd_line.py | 4 + third_party/python/Python/finalize.c | 1 + third_party/python/Python/import.c | 357 +++++++++++++++--- third_party/python/Python/pylifecycle.c | 1 + third_party/python/python.mk | 1 + 10 files changed, 340 insertions(+), 84 deletions(-) diff --git a/third_party/python/Include/import.h b/third_party/python/Include/import.h index 5e844880aa8..a2509e1a5a8 100644 --- a/third_party/python/Include/import.h +++ b/third_party/python/Include/import.h @@ -73,6 +73,8 @@ PyObject * PyImport_GetImporter(PyObject *path); PyObject * PyImport_Import(PyObject *name); PyObject * PyImport_ReloadModule(PyObject *m); void PyImport_Cleanup(void); +void _PyImportLookupTables_Init(void); +void _PyImportLookupTables_Cleanup(void); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 int PyImport_ImportFrozenModuleObject( PyObject *name diff --git a/third_party/python/Lib/importlib/_bootstrap.py b/third_party/python/Lib/importlib/_bootstrap.py index 7ff1415a2d1..9ae8032dbbf 100644 --- a/third_party/python/Lib/importlib/_bootstrap.py +++ b/third_party/python/Lib/importlib/_bootstrap.py @@ -226,7 +226,7 @@ def _verbose_message(message, *args, verbosity=1): def _requires_builtin(fxn): """Decorator to verify the named module is built-in.""" def _requires_builtin_wrapper(self, fullname): - if fullname not in BUILTIN_MODULE_NAMES: + if not _imp.is_builtin(fullname): raise ImportError('{!r} is not a built-in module'.format(fullname), name=fullname) return fxn(self, fullname) @@ -631,7 +631,7 @@ def module_repr(module): def find_spec(cls, fullname, path=None, target=None): if path is not None: return None - if fullname in BUILTIN_MODULE_NAMES: + if _imp.is_builtin(fullname): return spec_from_loader(fullname, cls, origin='built-in') else: return None @@ -651,7 +651,7 @@ def find_module(cls, fullname, path=None): @classmethod def create_module(self, spec): """Create a built-in module""" - if spec.name not in BUILTIN_MODULE_NAMES: + if not _imp.is_builtin(spec.name): raise ImportError('{!r} is not a built-in module'.format(spec.name), name=spec.name) return _call_with_frames_removed(_imp.create_builtin, spec) @@ -871,7 +871,7 @@ def _find_and_load_unlocked(name, import_): msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent) raise ModuleNotFoundError(msg, name=name) from None spec = _find_spec(name, path) - if spec is None and name in BUILTIN_MODULE_NAMES: + if spec is None and _imp.is_builtin(name): # If this module is a C extension, the interpreter # expects it to be a shared object located in path, # and returns spec is None because it was not found. @@ -1033,6 +1033,21 @@ def _builtin_from_name(name): raise ImportError('no built-in module named ' + name) return _load_unlocked(spec) +def _get_builtin_spec(name): + # called from CosmoImporter in import.c + return ModuleSpec(name, BuiltinImporter, origin="built-in", is_package=False) + +def _get_frozen_spec(name, is_package): + # called from CosmoImporter in import.c + return ModuleSpec(name, FrozenImporter, origin="frozen", is_package=is_package) + +def _get_zipstore_spec(name, loader, origin, is_package): + # called from CosmoImporter in import.c + spec = ModuleSpec(name, loader, origin=origin, is_package=is_package) + spec.has_location = True + if is_package: + spec.submodule_search_locations = [origin.rpartition("/")[0]] + return spec def _setup(sys_module, _imp_module): """Setup importlib by importing needed built-in modules and injecting them @@ -1042,16 +1057,15 @@ def _setup(sys_module, _imp_module): modules, those two modules must be explicitly passed in. """ - global _imp, sys, BUILTIN_MODULE_NAMES + global _imp, sys _imp = _imp_module sys = sys_module - BUILTIN_MODULE_NAMES = frozenset(sys.builtin_module_names) # Set up the spec for existing builtin/frozen modules. module_type = type(sys) for name, module in sys.modules.items(): if isinstance(module, module_type): - if name in BUILTIN_MODULE_NAMES: + if _imp.is_builtin(name): loader = BuiltinImporter elif _imp.is_frozen(name): loader = FrozenImporter @@ -1072,8 +1086,9 @@ def _install(sys_module, _imp_module): """Install importlib as the implementation of import.""" _setup(sys_module, _imp_module) - sys.meta_path.append(BuiltinImporter) - sys.meta_path.append(FrozenImporter) + sys.meta_path.append(_imp_module.CosmoImporter) + # sys.meta_path.append(BuiltinImporter) + # sys.meta_path.append(FrozenImporter) global _bootstrap_external import _frozen_importlib_external diff --git a/third_party/python/Lib/importlib/_bootstrap_external.py b/third_party/python/Lib/importlib/_bootstrap_external.py index 43fca240a4d..f8ab20b5448 100644 --- a/third_party/python/Lib/importlib/_bootstrap_external.py +++ b/third_party/python/Lib/importlib/_bootstrap_external.py @@ -1346,10 +1346,10 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + # extensions = ExtensionFileLoader, _imp.extension_suffixes() source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [bytecode, extensions, source] + return [source, bytecode] #, extensions] def _setup(_bootstrap_module): """Setup the path-based importers for importlib by importing needed diff --git a/third_party/python/Lib/io.py b/third_party/python/Lib/io.py index 968ee5073df..fbd0fe7d294 100644 --- a/third_party/python/Lib/io.py +++ b/third_party/python/Lib/io.py @@ -92,8 +92,6 @@ class TextIOBase(_io._TextIOBase, IOBase): del klass try: - from _io import _WindowsConsoleIO -except ImportError: + RawIOBase.register(_io._WindowsConsoleIO) +except AttributeError: pass -else: - RawIOBase.register(_WindowsConsoleIO) diff --git a/third_party/python/Lib/ntpath.py b/third_party/python/Lib/ntpath.py index 406f0441aaa..bcf83f09c7a 100644 --- a/third_party/python/Lib/ntpath.py +++ b/third_party/python/Lib/ntpath.py @@ -20,6 +20,7 @@ import os import sys import stat +import posix import genericpath from genericpath import * @@ -276,8 +277,8 @@ def lexists(path): # common case: drive letter roots. The alternative which uses GetVolumePathName # fails if the drive letter is the result of a SUBST. try: - from posix import _getvolumepathname -except ImportError: + _getvolumepathname = posix._getvolumepathname +except AttributeError: _getvolumepathname = None def ismount(path): """Test whether a path is a mount point (a drive root, the root of a @@ -534,9 +535,9 @@ def _abspath_fallback(path): # Return an absolute path. try: - from posix import _getfullpathname + _getfullpathname = posix._getfullpathname -except ImportError: # not running on Windows - mock up something sensible +except AttributeError: # not running on Windows - mock up something sensible abspath = _abspath_fallback else: # use native Windows method on Windows @@ -664,7 +665,7 @@ def commonpath(paths): # GetFinalPathNameByHandle is available starting with Windows 6.0. # Windows XP and non-Windows OS'es will mock _getfinalpathname. if sys.getwindowsversion()[:2] >= (6, 0): - from posix import _getfinalpathname + _getfinalpathname = posix._getfinalpathname else: raise ImportError except (AttributeError, ImportError, OSError): @@ -681,7 +682,7 @@ def _getfinalpathname(f): # attribute to tell whether or not the path is a directory. # This is overkill on Windows - just pass the path to GetFileAttributes # and check the attribute from there. - from posix import _isdir as isdir -except ImportError: + isdir = posix._isdir +except AttributeError: # Use genericpath.isdir as imported above. pass diff --git a/third_party/python/Lib/test/test_cmd_line.py b/third_party/python/Lib/test/test_cmd_line.py index 71901fab12f..83c0e5d8720 100644 --- a/third_party/python/Lib/test/test_cmd_line.py +++ b/third_party/python/Lib/test/test_cmd_line.py @@ -483,6 +483,10 @@ def test_isolatedmode(self): with open(fake, "w") as f: f.write("raise RuntimeError('isolated mode test')\n") with open(main, "w") as f: + f.write("import sys\n") + f.write("import _imp\n") + f.write("if sys.meta_path[0] == _imp.CosmoImporter:\n") + f.write("\tsys.meta_path.pop(0)\n") f.write("import uuid\n") f.write("print('ok')\n") self.assertRaises(subprocess.CalledProcessError, diff --git a/third_party/python/Python/finalize.c b/third_party/python/Python/finalize.c index b68b910c077..81d45ad9948 100644 --- a/third_party/python/Python/finalize.c +++ b/third_party/python/Python/finalize.c @@ -96,6 +96,7 @@ Py_FinalizeEx(void) #endif /* Destroy all modules */ PyImport_Cleanup(); + _PyImportLookupTables_Cleanup(); /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ if (_Py_FlushStdFiles() < 0) { diff --git a/third_party/python/Python/import.c b/third_party/python/Python/import.c index e62c24aa85a..c89a13109fb 100644 --- a/third_party/python/Python/import.c +++ b/third_party/python/Python/import.c @@ -4,11 +4,13 @@ │ Python 3 │ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/alg/alg.h" #include "libc/bits/bits.h" #include "libc/calls/calls.h" #include "libc/calls/struct/stat.h" #include "libc/calls/struct/stat.macros.h" #include "libc/fmt/conv.h" +#include "libc/macros.internal.h" #include "libc/runtime/gc.h" #include "libc/x/x.h" #include "libc/sysv/consts/o.h" @@ -82,9 +84,6 @@ static PyObject *extensions = NULL; static PyObject *initstr = NULL; -static struct stat stinfo; -_Py_IDENTIFIER(__builtins__); -_Py_IDENTIFIER(_load_module_shim); /*[clinic input] module _imp [clinic start generated code]*/ @@ -93,6 +92,27 @@ module _imp #include "third_party/python/Python/clinic/import.inc" /* Initialize things */ +typedef struct { + const char *name; + union { + const struct _inittab *tab; + const struct _frozen *frz; + }; +} initentry; + +typedef struct { + size_t n; + initentry *entries; +} Lookup; + +static Lookup Builtins_Lookup = {.n = 0, .entries = NULL}; +static Lookup Frozens_Lookup = {.n = 0, .entries = NULL}; + +static int cmp_initentry(const void *_x, const void *_y) { + const initentry *x = _x; + const initentry *y = _y; + return strcmp(x->name, y->name); +} void _PyImport_Init(void) @@ -106,6 +126,41 @@ _PyImport_Init(void) Py_FatalError("Can't backup builtins dict"); } +void _PyImportLookupTables_Init(void) { + size_t i, n; + if (Builtins_Lookup.entries == NULL) { + for(n=0; PyImport_Inittab[n].name; n++); + Builtins_Lookup.n = n; + Builtins_Lookup.entries = malloc(sizeof(initentry) * n); + for(i=0; i < n; i++) { + Builtins_Lookup.entries[i].name = PyImport_Inittab[i].name; + Builtins_Lookup.entries[i].tab = &(PyImport_Inittab[i]); + } + qsort(Builtins_Lookup.entries, Builtins_Lookup.n, sizeof(initentry), cmp_initentry); + } + if (Frozens_Lookup.entries == NULL) { + for(n=0; PyImport_FrozenModules[n].name; n++); + Frozens_Lookup.n = n; + Frozens_Lookup.entries = malloc(sizeof(initentry) * n); + for(i=0; itab->initfunc == NULL) return -1; + return 1; } return 0; } @@ -1095,6 +1152,8 @@ _imp_create_builtin(PyObject *module, PyObject *spec) /*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ { struct _inittab *p; + initentry key; + initentry *res; PyObject *name; char *namestr; PyObject *mod; @@ -1117,38 +1176,41 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - for (p = PyImport_Inittab; p->name != NULL; p++) { + key.name = namestr; + key.tab = NULL; + res = bsearch(&key, Builtins_Lookup.entries, Builtins_Lookup.n, sizeof(initentry), cmp_initentry); + + if (res != NULL) { + p = res->tab; PyModuleDef *def; - if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - mod = PyImport_AddModule(namestr); - Py_DECREF(name); - return mod; - } - mod = (*p->initfunc)(); - if (mod == NULL) { + if (p->initfunc == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + mod = PyImport_AddModule(namestr); + Py_DECREF(name); + return mod; + } + mod = (*p->initfunc)(); + if (mod == NULL) { + Py_DECREF(name); + return NULL; + } + if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { + Py_DECREF(name); + return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); + } else { + /* Remember pointer to module init function. */ + def = PyModule_GetDef(mod); + if (def == NULL) { Py_DECREF(name); return NULL; } - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - Py_DECREF(name); - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } else { - /* Remember pointer to module init function. */ - def = PyModule_GetDef(mod); - if (def == NULL) { - Py_DECREF(name); - return NULL; - } - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name) < 0) { - Py_DECREF(name); - return NULL; - } + def->m_base.m_init = p->initfunc; + if (_PyImport_FixupExtensionObject(mod, name, name) < 0) { Py_DECREF(name); - return mod; + return NULL; } + Py_DECREF(name); + return mod; } } Py_DECREF(name); @@ -1161,18 +1223,20 @@ _imp_create_builtin(PyObject *module, PyObject *spec) static const struct _frozen * find_frozen(PyObject *name) { - const struct _frozen *p; + initentry key; + initentry *res; if (name == NULL) return NULL; - for (p = PyImport_FrozenModules; ; p++) { - if (p->name == NULL) - return NULL; - if (_PyUnicode_EqualToASCIIString(name, p->name)) - break; + key.name = PyUnicode_AsUTF8(name); + key.frz = NULL; + + res = bsearch(&key, Frozens_Lookup.entries, Frozens_Lookup.n, sizeof(initentry), cmp_initentry); + if (res && res->frz->name != NULL) { + return res->frz; } - return p; + return NULL; } static PyObject * @@ -2057,6 +2121,7 @@ dump buffer /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/ static PyObject *_check_path_mode(const char *path, uint32_t mode) { + struct stat stinfo; if (stat(path, &stinfo)) Py_RETURN_FALSE; if ((stinfo.st_mode & S_IFMT) == mode) Py_RETURN_TRUE; Py_RETURN_FALSE; @@ -2092,6 +2157,7 @@ static PyObject *_imp_path_isdir(PyObject *module, PyObject *arg) { PyDoc_STRVAR(_imp_path_isdir_doc, "check if path is dir"); static PyObject *_imp_calc_mode(PyObject *module, PyObject *arg) { + struct stat stinfo; Py_ssize_t n; const char *path; if (!PyArg_Parse(arg, "s#:_calc_mode", &path, &n)) return 0; @@ -2101,6 +2167,7 @@ static PyObject *_imp_calc_mode(PyObject *module, PyObject *arg) { PyDoc_STRVAR(_imp_calc_mode_doc, "return stat.st_mode of path"); static PyObject *_imp_calc_mtime_and_size(PyObject *module, PyObject *arg) { + struct stat stinfo; Py_ssize_t n; const char *path; if (!PyArg_Parse(arg, "z#:_calc_mtime_and_size", &path, &n)) return 0; @@ -2145,17 +2212,24 @@ static PyObject *_imp_write_atomic(PyObject *module, PyObject **args, Py_buffer data = {NULL, NULL}; uint32_t mode = 0666; int fd; + int failure = 0; if (!_PyArg_ParseStack(args, nargs, "s#y*|I:_write_atomic", &path, &n, &data, &mode)) - return 0; + goto end; mode &= 0666; - if ((fd = open(path, O_EXCL | O_CREAT | O_WRONLY, mode)) == -1 || - write(fd, data.buf, data.len) == -1) { - PyErr_Format(PyExc_OSError, ""); - if (data.obj) PyBuffer_Release(&data); - return 0; + if ((fd = open(path, O_CREAT | O_WRONLY | O_TRUNC, mode)) == -1) { + failure = 1; + PyErr_Format(PyExc_OSError, "failed to create file: %s\n", path); + goto end; + } + if (write(fd, data.buf, data.len) == -1) { + failure = 1; + PyErr_Format(PyExc_OSError, "failed to write to file: %s\n", path); + goto end; } +end: if (data.obj) PyBuffer_Release(&data); + if (failure) return 0; Py_RETURN_NONE; } PyDoc_STRVAR(_imp_write_atomic_doc, "atomic write to a file"); @@ -2290,6 +2364,7 @@ static PyObject *_imp_cache_from_source(PyObject *module, PyObject **args, Py_ss PyDoc_STRVAR(_imp_cache_from_source_doc, "given a .py filename, return .pyc"); static PyObject *_imp_source_from_cache(PyObject *module, PyObject *arg) { + struct stat stinfo; char *path = NULL; Py_ssize_t pathlen = 0; if (!PyArg_Parse(PyOS_FSPath(arg), "z#:source_from_cache", &path, &pathlen)) @@ -2309,10 +2384,12 @@ static PyObject *_imp_source_from_cache(PyObject *module, PyObject *arg) { PyDoc_STRVAR(_imp_source_from_cache_doc, "given a .pyc filename, return .py"); typedef struct { - PyObject_HEAD char *name; + PyObject_HEAD + char *name; char *path; Py_ssize_t namelen; Py_ssize_t pathlen; + Py_ssize_t present; } SourcelessFileLoader; static PyTypeObject SourcelessFileLoaderType; @@ -2327,6 +2404,7 @@ static SourcelessFileLoader *SFLObject_new(PyObject *cls, PyObject *args, obj->path = NULL; obj->namelen = 0; obj->pathlen = 0; + obj->present = 0; return obj; } @@ -2366,6 +2444,7 @@ static int SFLObject_init(SourcelessFileLoader *self, PyObject *args, // TODO: should this be via PyMem_RawMalloc? self->name = strndup(name, namelen); self->path = strndup(path, pathlen); + self->present = 0; } return result; } @@ -2401,6 +2480,7 @@ static PyObject *SFLObject_get_source(SourcelessFileLoader *self, } static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) { + struct stat stinfo; char bytecode_header[12] = {0}; int32_t magic = 0; size_t headerlen; @@ -2420,7 +2500,8 @@ static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) { self->name, name); goto exit; } - if (stat(self->path, &stinfo) || !(fp = fopen(self->path, "rb"))) { + self->present = self->present || !stat(self->path, &stinfo); + if (!self->present || !(fp = fopen(self->path, "rb"))) { PyErr_Format(PyExc_ImportError, "%s does not exist\n", self->path); goto exit; } @@ -2439,20 +2520,24 @@ static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) { "reached EOF while reading timestamp in %s\n", name); goto exit; } - if (headerlen < 12 || stinfo.st_size <= headerlen) { + if (headerlen < 12) { PyErr_Format(PyExc_ImportError, "reached EOF while size of source in %s\n", name); goto exit; } // return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path) + /* since we don't have the stat call sometimes, we need + * a different way to load the remaining bytes into file + */ + /* rawlen = stinfo.st_size - headerlen; rawbuf = PyMem_RawMalloc(rawlen); if (rawlen != fread(rawbuf, sizeof(char), rawlen, fp)) { PyErr_Format(PyExc_ImportError, "reached EOF while size of source in %s\n", name); goto exit; - } - if (!(res = PyMarshal_ReadObjectFromString(rawbuf, rawlen))) goto exit; + }*/ + if (!(res = PyMarshal_ReadObjectFromFile(fp))) goto exit; exit: if (rawbuf) PyMem_RawFree(rawbuf); if (fp) fclose(fp); @@ -2460,6 +2545,7 @@ static PyObject *SFLObject_get_code(SourcelessFileLoader *self, PyObject *arg) { } static PyObject *SFLObject_get_data(SourcelessFileLoader *self, PyObject *arg) { + struct stat stinfo; char *name = NULL; char *data = NULL; size_t datalen = 0; @@ -2491,6 +2577,7 @@ static PyObject *SFLObject_get_filename(SourcelessFileLoader *self, static PyObject *SFLObject_load_module(SourcelessFileLoader *self, PyObject **args, Py_ssize_t nargs) { + _Py_IDENTIFIER(_load_module_shim); char *name = NULL; PyObject *bootstrap = NULL; PyObject *fullname = NULL; @@ -2523,6 +2610,7 @@ static PyObject *SFLObject_create_module(SourcelessFileLoader *self, static PyObject *SFLObject_exec_module(SourcelessFileLoader *self, PyObject *arg) { + _Py_IDENTIFIER(__builtins__); PyObject *module = NULL; PyObject *name = NULL; PyObject *code = NULL; @@ -2598,10 +2686,8 @@ static PyMethodDef SFLObject_methods[] = { }; static PyTypeObject SourcelessFileLoaderType = { - /* The ob_type field must be initialized in the module init function - * to be portable to Windows without using C++. */ - PyVarObject_HEAD_INIT(NULL, 0).tp_name = - "_imp.SourcelessFileLoader", /*tp_name*/ + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_imp.SourcelessFileLoader", /*tp_name*/ .tp_basicsize = sizeof(SourcelessFileLoader), /*tp_basicsize*/ .tp_dealloc = (destructor)SFLObject_dealloc, /*tp_dealloc*/ .tp_hash = (hashfunc)SFLObject_hash, /*tp_hash*/ @@ -2612,6 +2698,145 @@ static PyTypeObject SourcelessFileLoaderType = { .tp_new = (newfunc)SFLObject_new, /*tp_new*/ }; +typedef struct { + PyObject_HEAD +} CosmoImporter; + +static PyTypeObject CosmoImporterType; +#define CosmoImporterCheck(o) (Py_TYPE(o) == &CosmoImporterType) + +static PyObject *CosmoImporter_find_spec(PyObject *cls, PyObject **args, + Py_ssize_t nargs, PyObject *kwargs) { + static const char *const _keywords[] = {"fullname", "path", "target", NULL}; + static _PyArg_Parser _parser = {"U|OO", _keywords, 0}; + _Py_IDENTIFIER(_get_builtin_spec); + _Py_IDENTIFIER(_get_frozen_spec); + _Py_IDENTIFIER(_get_zipstore_spec); + + PyObject *fullname = NULL; + PyObject *path = NULL; + /* path is a LIST! it contains strings similar to those in sys.path, + * ie folders that are likely to contain a particular file. + * during startup the expected scenario is checking the ZIP store + * of the APE, so we ignore path and let these slower cases to + * handled by the importer objects already provided by Python. */ + PyObject *target = NULL; + PyInterpreterState *interp = PyThreadState_GET()->interp; + + const struct _frozen *p = NULL; + + static const char basepath[] = "/zip/.python/"; + const char *cname = NULL; + Py_ssize_t cnamelen = 0; + + char *newpath = NULL; + Py_ssize_t newpathsize = 0; + Py_ssize_t newpathlen = 0; + Py_ssize_t i = 0; + + SourcelessFileLoader *loader = NULL; + PyObject *origin = NULL; + int inside_zip = 0; + int is_package = 0; + int is_available = 0; + + struct stat stinfo; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwargs, &_parser, &fullname, + &path, &target)) { + return NULL; + } + + if (fullname == NULL) { + PyErr_SetString(PyExc_ImportError, "fullname not provided\n"); + return NULL; + } + + if ((!path || path == Py_None)) { + /* we do some of BuiltinImporter's work */ + if (is_builtin(fullname) == 1) { + return _PyObject_CallMethodIdObjArgs( + interp->importlib, &PyId__get_builtin_spec, fullname, NULL); + } + /* we do some of FrozenImporter's work */ + else if ((p = find_frozen(fullname)) != NULL) { + return _PyObject_CallMethodIdObjArgs(interp->importlib, + &PyId__get_frozen_spec, fullname, + PyBool_FromLong(p->size < 0), NULL); + } + } + + if (!PyArg_Parse(fullname, "z#:find_spec", &cname, &cnamelen)) return 0; + /* before checking within the zip store, + * we can check cname here to skip any values + * of cname that we know for sure won't be there, + * because worst case is two failed stat calls here + */ + + newpathsize = sizeof(basepath) + cnamelen + sizeof("/__init__.pyc") + 1; + newpath = _gc(malloc(newpathsize)); + bzero(newpath, newpathsize); + /* performing a memccpy sequence equivalent to: + * snprintf(newpath, newpathsize, "/zip/.python/%s.pyc", cname); */ + memccpy(newpath, basepath, '\0', newpathsize); + memccpy(newpath + sizeof(basepath) - 1, cname, '\0', + newpathsize - sizeof(basepath)); + memccpy(newpath + sizeof(basepath) + cnamelen - 1, ".pyc", '\0', + newpathsize - (sizeof(basepath) + cnamelen)); + + /* if cname part of newpath has '.' (e.g. encodings.utf_8) convert them to '/' + */ + for (i = sizeof(basepath); i < sizeof(basepath) + cnamelen - 1; i++) { + if (newpath[i] == '.') newpath[i] = '/'; + } + + is_available = inside_zip || !stat(newpath, &stinfo); + if (is_package || !is_available) { + memccpy(newpath + sizeof(basepath) + cnamelen - 1, "/__init__.pyc", '\0', + newpathsize); + is_available = is_available || !stat(newpath, &stinfo); + is_package = 1; + } + + if (is_available) { + newpathlen = strlen(newpath); + loader = SFLObject_new(NULL, NULL, NULL); + origin = PyUnicode_FromStringAndSize(newpath, newpathlen); + if (loader == NULL || origin == NULL) { + return NULL; + } + loader->name = strdup(cname); + loader->namelen = cnamelen; + loader->path = strdup(newpath); + loader->pathlen = newpathlen; + loader->present = 1; /* this means we avoid atleast one stat call (the one + in SFLObject_get_code) */ + return _PyObject_CallMethodIdObjArgs(interp->importlib, + &PyId__get_zipstore_spec, fullname, + (PyObject *)loader, (PyObject *)origin, + PyBool_FromLong(is_package), NULL); + } + + Py_RETURN_NONE; +} + +static PyMethodDef CosmoImporter_methods[] = { + {"find_spec", (PyCFunction)CosmoImporter_find_spec, + METH_FASTCALL | METH_KEYWORDS | METH_CLASS, PyDoc_STR("")}, + {NULL, NULL} // sentinel +}; + +static PyTypeObject CosmoImporterType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_imp.CosmoImporter", /* tp_name */ + .tp_dealloc = 0, + .tp_basicsize = sizeof(CosmoImporter), /* tp_basicsize */ + .tp_flags = Py_TPFLAGS_DEFAULT, /* tp_flags */ + .tp_methods = CosmoImporter_methods, /* tp_methods */ + .tp_init = 0, + .tp_new = 0, +}; + PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib and imp."); @@ -2673,8 +2898,16 @@ PyInit_imp(void) if (PyType_Ready(&SourcelessFileLoaderType) < 0) goto failure; - PyModule_AddObject(m, "SourcelessFileLoader", (PyObject*)&SourcelessFileLoaderType); - + if (PyModule_AddObject(m, "SourcelessFileLoader", (PyObject*)&SourcelessFileLoaderType) < 0) + goto failure; + if (PyType_Ready(&CosmoImporterType) < 0) + goto failure; + if (PyModule_AddObject(m, "CosmoImporter", (PyObject*)&CosmoImporterType) < 0) + goto failure; + /* test_atexit segfaults without the below incref, but + * I'm not supposed to Py_INCREF a static PyTypeObject, so + * what's going on? */ + Py_INCREF(&CosmoImporterType); return m; failure: Py_XDECREF(m); diff --git a/third_party/python/Python/pylifecycle.c b/third_party/python/Python/pylifecycle.c index 546c9361e4f..3f5a34ef349 100644 --- a/third_party/python/Python/pylifecycle.c +++ b/third_party/python/Python/pylifecycle.c @@ -248,6 +248,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) PySys_SetObject("__stderr__", pstderr); Py_DECREF(pstderr); + _PyImportLookupTables_Init(); _PyImport_Init(); _PyImportHooks_Init(); diff --git a/third_party/python/python.mk b/third_party/python/python.mk index 757e054fbf9..4e1adf81b76 100644 --- a/third_party/python/python.mk +++ b/third_party/python/python.mk @@ -441,6 +441,7 @@ THIRD_PARTY_PYTHON_STAGE1_A_SRCS = \ THIRD_PARTY_PYTHON_STAGE1_A_DIRECTDEPS = \ DSP_SCALE \ + LIBC_ALG \ LIBC_BITS \ LIBC_CALLS \ LIBC_FMT \