Initial commit

.ci/pythonPackagesToInstallFromGit.txt

@@ -0,0 +1,5 @@
+https://github.com/UniGrammar/UniGrammarRuntimeCore.py
+https://github.com/UniGrammar/UniGrammarRuntime.py
+https://github.com/igordejanovic/parglare.git
+https://github.com/neogeny/TatSu.git
+https://github.com/erikrose/parsimonious.git

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml,yug}]
+indent_style = space
+indent_size = 2
+
+[grammars/*.txt]
+insert_final_newline = false

.github/.templateMarker

@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"

.github/workflows/CI.yml

@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,24 @@
+/grammars/apt.interp
+/grammars/apt.tokens
+/grammars/aptLexer.*
+/grammars/*.py
+/grammars/*.jar
+/grammars/*.java
+/grammars/*.class
+/grammars/grammar.pgt
+
+__pycache__
+*.pyc
+*.pyo
+/*.egg-info
+*.srctrlbm
+*.srctrldb
+build
+dist
+.eggs
+monkeytype.sqlite3
+/.ipynb_checkpoints
+
+.ninja_log
+.ninja_deps
+/tests/lan_proto_test

.gitlab-ci.yml

@@ -0,0 +1,55 @@
+image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest
+
+variables:
+  DOCKER_DRIVER: overlay2
+  SAST_ANALYZER_IMAGE_TAG: latest
+  SAST_DISABLE_DIND: "true"
+  SAST_CONFIDENCE_LEVEL: 5
+  CODECLIMATE_VERSION: latest
+
+include:
+  - template: SAST.gitlab-ci.yml
+  - template: Code-Quality.gitlab-ci.yml
+
+build:
+  tags:
+    - shared
+    - linux
+  stage: build
+  interruptible: true
+  variables:
+    GIT_DEPTH: "1"
+    PYTHONUSERBASE: "${CI_PROJECT_DIR}/python_user_packages"
+
+  before_script:
+    - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables`
+    #- git clone --depth=1 --filter=sparse:path=src/python https://github.com/waxeye-org/waxeye.git
+    - git clone --depth=1 https://github.com/waxeye-org/waxeye.git
+    - cd ./waxeye/src/python
+    - python3 ./setup.py  bdist_wheel
+    - pip3 install --upgrade ./dist/*.whl
+    - cd ../../../
+
+  cache:
+    paths:
+      - /usr/local/site-packages
+      - /usr/local/lib/python*/site-packages
+
+  script:
+    - python3 setup.py bdist_wheel
+    - pip3 install --user --upgrade ./dist/*.whl
+    - cd ./tests
+    #- coverage run -a --branch --source=AptSourcesList -m pytest --junitxml=./rspec.xml --forked ./test*.py
+    #- coverage report -m || true
+    #- coveralls || true
+    #- codecov || true
+    #- cd ..
+    - mkdir wheels
+    - mv ./dist/*.whl ./wheels/AptSourcesList-0.CI-py3-none-any.whl
+
+  artifacts:
+    paths:
+      - wheels
+      - $PYTHONUSERBASE
+    reports:
+      junit: ./rspec.xml

AptSourcesList/Record.py

@@ -0,0 +1,29 @@
+class Record:
+	"""Represents a single line in sources.list mentioning a source."""
+
+	__slots__ = ("commented", "type", "options", "schemaModifiers", "uri", "distribution", "components")
+
+	def __init__(self):
+		self.commented = None
+		self.type = None
+		self.options = None
+		self.uri = None
+		self.distribution = None
+		self.components = None
+		self.schemaModifiers = None
+
+	def serializeOptions(self) -> str:
+		if self.options:
+			return "[" + ",".join(k + "=" + v for k, v in self.options.items()) + "]"
+		return ""
+
+	def serializeComponents(self) -> str:
+		if self.components:
+			return " ".join(self.components)
+		return ""
+
+	def __str__(self) -> str:
+		return " ".join(filter(None, ("#" if self.commented else "", self.type, self.serializeOptions(), "+".join(self.schemaModifiers + [self.uri,]), self.distribution, self.serializeComponents())))
+
+	def __repr__(self) -> str:
+		return self.__class__.__name__ + "(" + repr(str(self)) + ")"

AptSourcesList/RecordParser.py

@@ -0,0 +1,81 @@
+import typing
+from abc import ABC, abstractmethod
+from .Record import Record
+
+
+class RecordParser(ABC):
+	"""A class commanding the parsing. Calls the generated parser and postporcesses its output"""
+
+	__slots__ = ()
+
+	EX_CLASS = Exception
+
+	@abstractmethod
+	def iterateList(self, lst):
+		raise NotImplementedError()
+
+	@abstractmethod
+	def isList(self, lst):
+		raise NotImplementedError()
+
+	@abstractmethod
+	def parseURI(self, rec, uri):
+		raise NotImplementedError()
+
+	def mergeShit(self, lst: typing.Any) -> str:
+		res = []
+		stack = []
+		if self.isList(lst):
+			for t in self.iterateList(lst):
+				res.append(self.mergeShit(t))
+		else:
+			res.append(self.getTextFromToken(lst))
+		return "".join(res)
+
+	def descendIntoURISchema(self, schema: typing.Any) -> typing.Iterator[str]:
+		restWords = schema
+		yield self.getTextFromToken(schema.word)
+		if schema.restWords:
+			for w in self.iterateList(schema.restWords):
+				res = self.getTextFromToken(w.word)
+				yield res
+
+	def __call__(self, s: str) -> Record:
+		rec = Record()
+		parsed = self.parse(s)
+
+		try:
+			commented = bool(self.getTextFromToken(parsed.commented))
+		except:
+			commented = False
+
+		rec.commented = commented
+		rec.type = self.getTextFromToken(parsed.rType)
+		rec.options = parsed.options
+		rec.uri = parsed.uri
+		rec.distribution = self.mergeShit(parsed.distribution)
+		rec.components = parsed.components
+		rec.schemaModifiers = None
+
+		if rec.uri:
+			rec.uri = self.parseURI(rec, rec.uri)
+
+		if rec.options:
+			if rec.options.pairs:
+				pairs = rec.options.pairs
+				res = [pairs.firstOption]
+
+				try:
+					restOptions = pairs.restOptions
+				except:
+					restOptions = []
+
+				for p in self.iterateList(restOptions):
+					res.append(p.option)
+
+				rec.options = res
+				rec.options = {self.getTextFromToken(p.key): self.mergeShit(p.value) for p in rec.options}
+
+		if rec.components:
+			rec.components = [self.mergeShit(c.cId) for c in self.iterateList(rec.components)]
+		return rec

AptSourcesList/__init__.py

@@ -0,0 +1,62 @@
+__all__ = ("parseLine", "parseSourceList")
+
+import typing
+from pathlib import Path
+import _io
+
+from UniGrammarRuntime.ParserBundle import ParserBundle
+
+
+thisFile = Path(__file__).absolute()
+thisDir = thisFile.parent
+bundleDir = thisDir / "parserBundle"
+
+bundle = ParserBundle(bundleDir)
+
+grammar = bundle.grammars["apt_source"]
+wrapper = grammar.getWrapper("parglare")
+
+
+parseLine = wrapper
+
+
+defaultSourceList = None
+
+
+def getDefaultSourceList():
+	global defaultSourceList
+
+	if defaultSourceList is not None:
+		return defaultSourceList
+
+	try:
+		from fuckapt import sourcesListFile
+
+		defaultSourceList = sourcesListFile
+	except ImportError:
+		defaultSourceList = Path("/etc/apt/sources.list")
+
+	return defaultSourceList
+
+
+def parseSourceList(lst: typing.Union[str, Path, _io._TextIOBase] = None):
+	"""Parses whole `sources.list`"""
+	if lst is None:
+		lst = getDefaultSourceList()
+
+	if isinstance(lst, str):
+		lines = lst.splitlines()
+	elif isinstance(lst, Path):
+		lines = lst.open("rt", encoding="utf-8")
+	elif isinstance(lst, _io._TextIOBase):
+		lines = lst
+	else:
+		raise ValueError("`lst` has wrong type")
+
+	for l in lines:
+		l = l.strip()
+		if l:
+			parsed = parseLine(l)
+			yield parsed
+		else:
+			yield ""

AptSourcesList/__main__.py

@@ -0,0 +1,5 @@
+if __name__ == "__main__":
+	from . import parseSourceList
+	from pprint import pprint
+
+	pprint(list(parseSourceList()))