From 187e72fd5563e18541a056bf3d29545f42828846 Mon Sep 17 00:00:00 2001 From: Jan Gantzert Date: Sat, 14 Sep 2024 10:25:35 +0200 Subject: [PATCH 1/2] add submit proxy type --- src/Composer/PacketonRepositoryFactory.php | 2 + .../Repository/ComposerProxyRepository.php | 46 +++++++++ src/Filter/VersionFilter.php | 25 +++++ src/Form/Type/Package/BasePackageType.php | 1 + src/Form/Type/Package/ProxyPackageType.php | 93 +++++++++++++++++++ src/Model/ComposerProxyPackageManager.php | 58 ++++++++++++ src/Package/RepTypes.php | 7 +- src/Package/Updater.php | 32 ++++++- src/Service/DistManager.php | 6 ++ .../Constraint/PackageRepositoryValidator.php | 1 + 10 files changed, 265 insertions(+), 6 deletions(-) create mode 100644 src/Composer/Repository/ComposerProxyRepository.php create mode 100644 src/Filter/VersionFilter.php create mode 100644 src/Form/Type/Package/ProxyPackageType.php create mode 100644 src/Model/ComposerProxyPackageManager.php diff --git a/src/Composer/PacketonRepositoryFactory.php b/src/Composer/PacketonRepositoryFactory.php index 23167554..eda332cc 100644 --- a/src/Composer/PacketonRepositoryFactory.php +++ b/src/Composer/PacketonRepositoryFactory.php @@ -9,6 +9,7 @@ use Composer\IO\IOInterface; use Doctrine\Persistence\ManagerRegistry; use Packeton\Composer\Repository\ArtifactRepository; +use Packeton\Composer\Repository\ComposerProxyRepository; use Packeton\Composer\Repository\CustomJsonRepository; use Packeton\Composer\Repository\PacketonRepositoryInterface; use Packeton\Composer\Repository\VcsRepository; @@ -42,6 +43,7 @@ public function create(array $repoConfig, IOInterface $io, Config $config, ?stri return match ($type) { RepTypes::ARTIFACT => new ArtifactRepository($repoConfig, $this->zipballStorage, $this->registry, $io, $config, $httpDownloader), RepTypes::CUSTOM, RepTypes::VIRTUAL => new CustomJsonRepository($repoConfig, $this->registry, $io, $config, $httpDownloader), + RepTypes::PROXY => new ComposerProxyRepository($repoConfig, $io, $config, $httpDownloader), default => new VcsRepository($repoConfig, $io, $config, $httpDownloader, $this->driverFactory, null, $process), }; } diff --git a/src/Composer/Repository/ComposerProxyRepository.php b/src/Composer/Repository/ComposerProxyRepository.php new file mode 100644 index 00000000..3657c7d0 --- /dev/null +++ b/src/Composer/Repository/ComposerProxyRepository.php @@ -0,0 +1,46 @@ +process ??= new ProcessExecutor($this->io); + } + + public function getHttpDownloader(): HttpDownloader + { + return $this->httpDownloader; + } + + public function getProcessExecutor(): ProcessExecutor + { + return $this->process; + } + + public function getConfig(): Config + { + return $this->config; + } + + public function getIO(): IOInterface + { + return $this->io; + } +} diff --git a/src/Filter/VersionFilter.php b/src/Filter/VersionFilter.php new file mode 100644 index 00000000..4b9639d7 --- /dev/null +++ b/src/Filter/VersionFilter.php @@ -0,0 +1,25 @@ +getName() === $repoName) { + $result[] = $version; + } + } + return $result; + } +} \ No newline at end of file diff --git a/src/Form/Type/Package/BasePackageType.php b/src/Form/Type/Package/BasePackageType.php index 997b48a0..f4c36b32 100644 --- a/src/Form/Type/Package/BasePackageType.php +++ b/src/Form/Type/Package/BasePackageType.php @@ -27,6 +27,7 @@ public function buildForm(FormBuilderInterface $builder, array $options): void 'MonoRepos (only GIT)' => RepTypes::MONO_REPO, 'Artifacts' => RepTypes::ARTIFACT, 'Custom (JSON)' => RepTypes::CUSTOM, + 'Proxy Repo' => RepTypes::PROXY, 'Virtual (only JSON metadata)' => RepTypes::VIRTUAL, 'Satis / Packagist.com / VCS Import' => 'import', // only redirect ]; diff --git a/src/Form/Type/Package/ProxyPackageType.php b/src/Form/Type/Package/ProxyPackageType.php new file mode 100644 index 00000000..71d6a540 --- /dev/null +++ b/src/Form/Type/Package/ProxyPackageType.php @@ -0,0 +1,93 @@ +packageManager = $packageManager; + } + + /** + * {@inheritdoc} + */ + public function buildForm(FormBuilderInterface $builder, array $options): void + { + $builder + ->add('credentials', CredentialType::class) + ->add('name', TextType::class, [ + 'required' => true, + 'constraints' => [new NotBlank()], + 'attr' => ['class' => 'package-repo-info', 'placeholder' => 'acme/package-name'], + 'disabled' => false === $options['is_created'], + ]) + ->add('repository', TextType::class, [ + 'label' => 'Packages.json', + 'attr' => [ + 'class' => 'package-repo-info', + 'placeholder' => 'e.g.: https://repo.magento.com/packages.json', + ], + ]); + + $builder->addEventListener(FormEvents::POST_SUBMIT, [$this, 'updateRepository'], 255); + } + + /** + * {@inheritdoc} + */ + public function getParent(): string + { + return BasePackageType::class; + } + + /** + * @param FormEvent $event + */ + public function updateRepository(FormEvent $event): void + { + $package = $event->getData(); + if ($package instanceof Package) { + $this->packageManager->updatePackageUrl($package); + } + } + + /** + * {@inheritdoc} + */ + public function configureOptions(OptionsResolver $resolver): void + { + $resolver->setDefaults([ + 'data_class' => Package::class, + ]); + } + + /** + * {@inheritdoc} + */ + public function getBlockPrefix(): string + { + return 'proxy'; + } +} diff --git a/src/Model/ComposerProxyPackageManager.php b/src/Model/ComposerProxyPackageManager.php new file mode 100644 index 00000000..04341d9d --- /dev/null +++ b/src/Model/ComposerProxyPackageManager.php @@ -0,0 +1,58 @@ +getVersionByReference($reference) ?: $package->getVersions()->first(); + if (null === $version) { + throw new \RuntimeException("Not found any versions for reference '$reference' of package '{$package->getName()}'"); + } + + $keyName = $this->config->buildName($package->getName(), $version->getReference(), $version->getVersion()); + $cachedName = $this->config->resolvePath($keyName); + if (file_exists($cachedName)) { + return $cachedName; + } + + $selected = []; + $serialized = $package->getCustomVersions(); + foreach ($serialized as $data) { + $verName = $data['version'] ?? null; + if ($verName === $version->getVersion() || $verName === $version->getNormalizedVersion()) { + $selected = $data['definition'] ?? []; + $selected['version'] = $version->getVersion(); + } + } + + $selected['name'] = $package->getName(); + $dir = dirname($cachedName); + if (!is_dir($dir)) { + @mkdir($dir, 0777, true); + } + + $url = $package->getVersionByReference($reference)->getDist()['proxy_url']; + $response = $repository->getHttpDownloader()->get($url); + $body = (string) $response->getBody(); + + $this->baseStorage->write($keyName, $body); + + return $cachedName; + } + +} diff --git a/src/Package/RepTypes.php b/src/Package/RepTypes.php index de536f51..bf8f19da 100644 --- a/src/Package/RepTypes.php +++ b/src/Package/RepTypes.php @@ -9,6 +9,7 @@ use Packeton\Form\Type\Package\IntegrationPackageType; use Packeton\Form\Type\Package\MonoRepoPackageType; use Packeton\Form\Type\Package\PackageType; +use Packeton\Form\Type\Package\ProxyPackageType; class RepTypes { @@ -18,6 +19,7 @@ class RepTypes public const INTEGRATION = 'integration'; public const CUSTOM = 'custom'; public const VIRTUAL = 'virtual'; + public const PROXY = 'proxy'; private static $types = [ self::ARTIFACT, @@ -26,6 +28,7 @@ class RepTypes self::VCS, self::CUSTOM, self::VIRTUAL, + self::PROXY, ]; public static function getFormType(?string $type): string @@ -35,6 +38,7 @@ public static function getFormType(?string $type): string self::ARTIFACT => ArtifactPackageType::class, self::INTEGRATION => IntegrationPackageType::class, self::CUSTOM, self::VIRTUAL => CustomPackageType::class, + self::PROXY => ProxyPackageType::class, default => PackageType::class, }; } @@ -47,7 +51,7 @@ public static function isNotAutoCrawled(): array public static function isBuildInDist(?string $type): bool { return match ($type) { - self::ARTIFACT, self::CUSTOM, self::VIRTUAL => true, + self::ARTIFACT, self::CUSTOM, self::VIRTUAL => true, self::PROXY => true, default => false, }; } @@ -69,6 +73,7 @@ public static function normalizeType(?string $type): string self::INTEGRATION => self::INTEGRATION, self::CUSTOM => self::CUSTOM, self::VIRTUAL => self::VIRTUAL, + self::PROXY => self::PROXY, default => self::VCS, }; } diff --git a/src/Package/Updater.php b/src/Package/Updater.php index f879c7a4..925f286e 100644 --- a/src/Package/Updater.php +++ b/src/Package/Updater.php @@ -33,6 +33,7 @@ use Doctrine\ORM\EntityManagerInterface; use Doctrine\Persistence\ManagerRegistry; use Packeton\Composer\PackagistFactory; +use Packeton\Composer\Repository\ComposerProxyRepository; use Packeton\Composer\Repository\PacketonRepositoryInterface; use Packeton\Entity\Author; use Packeton\Entity\Package; @@ -41,6 +42,7 @@ use Packeton\Entity\SuggestLink; use Packeton\Event\SecurityAdvisoryEvent; use Packeton\Event\UpdaterEvent; +use Packeton\Mirror\Service\ProxyHttpDownloader; use Packeton\Model\ProviderManager; use Packeton\Repository\VersionRepository; use Packeton\Service\DistConfig; @@ -95,6 +97,7 @@ public function __construct( protected ProviderManager $providerManager, protected EventDispatcherInterface $dispatcher, protected DistManager $distManager, + protected ProxyHttpDownloader $downloader, ) { ErrorHandler::register(); } @@ -109,7 +112,7 @@ public function setSerializerCachePath(?string $serializerCachePath): void */ public static function supportRepoTypes(): iterable { - return [RepTypes::VCS, RepTypes::ARTIFACT, RepTypes::INTEGRATION, RepTypes::CUSTOM, RepTypes::VIRTUAL]; + return [RepTypes::VCS, RepTypes::ARTIFACT, RepTypes::INTEGRATION, RepTypes::CUSTOM, RepTypes::VIRTUAL, RepTypes::PROXY]; } /** @@ -134,7 +137,12 @@ public function update(IOInterface $io, Config $config, Package $package, Reposi $flags |= $addFlags; } - $versions = PacketonUtils::sort($repository->getPackages()); + if ($repository instanceof ComposerProxyRepository) { + $versions = PacketonUtils::sort($repository->findPackages($package->getName())); + } else { + $versions = PacketonUtils::sort($repository->getPackages()); + } + /** @var VersionRepository $versionRepository */ $versionRepository = $this->doctrine->getRepository(Version::class); if (null === $rootIdentifier && ($probe = end($versions))) { @@ -563,7 +571,7 @@ private function updateArchive(PackageInterface $data, Package $package): ?array // Process local path repos if (is_string($distUrl = $data->getDistUrl()) && (str_starts_with($distUrl, '/') || $distUrl === DistConfig::HOSTNAME_PLACEHOLDER) - && (empty($data->getSourceUrl()) || in_array($package->getRepoType(), [RepTypes::CUSTOM, RepTypes::VIRTUAL], true)) + && (empty($data->getSourceUrl()) || in_array($package->getRepoType(), [RepTypes::CUSTOM, RepTypes::VIRTUAL, RepTypes::PROXY], true)) ) { return [ 'url' => $this->distConfig->generateRoute($data->getName(), $data->getDistReference(), $data->getDistType()), @@ -609,8 +617,22 @@ private function updateArchive(PackageInterface $data, Package $package): ?array } $dist['type'] = $this->distConfig->getArchiveFormat(); - $dist['url'] = $this->distConfig->generateRoute($data->getName(), $data->getSourceReference()); - $dist['reference'] = $data->getSourceReference(); + + $url = $data->getSourceReference(); + if (null === $url) { + $url = $data->getDistReference(); + } + + if (null === $url) { + $url = sha1($data->getVersion()); + } + + $dist['url'] = $this->distConfig->generateRoute($data->getName(), $url); + $dist['reference'] = $url; + + if ($package->getRepoType() === RepTypes::PROXY) { + $dist['proxy_url'] = $data->getDistUrl(); + } return $dist; } diff --git a/src/Service/DistManager.php b/src/Service/DistManager.php index dbc83f1c..829201fa 100644 --- a/src/Service/DistManager.php +++ b/src/Service/DistManager.php @@ -18,6 +18,7 @@ use Packeton\Entity\Version; use Packeton\Integrations\IntegrationRegistry; use Packeton\Integrations\ZipballInterface; +use Packeton\Model\ComposerProxyPackageManager; use Packeton\Model\UploadZipballStorage; use Packeton\Model\VirtualPackageManager; use Packeton\Package\RepTypes; @@ -35,6 +36,7 @@ public function __construct( private readonly IntegrationRegistry $integrations, private readonly FilesystemOperator $baseStorage, private readonly Filesystem $fs, + private readonly ComposerProxyPackageManager $composerProxyPackageManager, private readonly VirtualPackageManager $virtualPackageManager, ) { } @@ -183,6 +185,10 @@ private function downloadArtifact(string $reference, Package $package): ?string $packages = $repository->getPackages(); $found = array_filter($packages, static fn($p) => $reference === $p->getDistReference()); + if ($package->getRepoType() === RepTypes::PROXY) { + return $this->composerProxyPackageManager->buildArchive($package, $repository, $reference); + } + /** @var PackageInterface $pkg */ if ($pkg = reset($found)) { $distUrl = $pkg->getDistUrl(); diff --git a/src/Validator/Constraint/PackageRepositoryValidator.php b/src/Validator/Constraint/PackageRepositoryValidator.php index 0146eac3..fa48aff8 100644 --- a/src/Validator/Constraint/PackageRepositoryValidator.php +++ b/src/Validator/Constraint/PackageRepositoryValidator.php @@ -35,6 +35,7 @@ public function validate(mixed $value, Constraint $constraint): void match ($value->getRepoType()) { RepTypes::ARTIFACT => $this->validateArtifactPackage($value), RepTypes::CUSTOM, RepTypes::VIRTUAL => $this->validateCustomPackage($value), + RepTypes::PROXY => null, default => $this->validateVcsPackage($value), }; } From 86c37ab880dd780e66984ff85ad50c2ec8a5fa0b Mon Sep 17 00:00:00 2001 From: Jan Gantzert Date: Thu, 10 Oct 2024 01:38:46 +0200 Subject: [PATCH 2/2] do not use package search for proxy repos --- src/Service/DistManager.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Service/DistManager.php b/src/Service/DistManager.php index 829201fa..5b544e30 100644 --- a/src/Service/DistManager.php +++ b/src/Service/DistManager.php @@ -13,6 +13,7 @@ use Doctrine\Persistence\ManagerRegistry; use League\Flysystem\FilesystemOperator; use Packeton\Composer\PackagistFactory; +use Packeton\Composer\Repository\ComposerProxyRepository; use Packeton\Composer\Repository\PacketonRepositoryInterface; use Packeton\Entity\Package; use Packeton\Entity\Version; @@ -182,8 +183,11 @@ private function downloadArtifact(string $reference, Package $package): ?string } $repository = $this->createRepositoryAndIo($package); - $packages = $repository->getPackages(); - $found = array_filter($packages, static fn($p) => $reference === $p->getDistReference()); + + if (!$repository instanceof ComposerProxyRepository) { + $packages = $repository->getPackages(); + $found = array_filter($packages, static fn($p) => $reference === $p->getDistReference()); + } if ($package->getRepoType() === RepTypes::PROXY) { return $this->composerProxyPackageManager->buildArchive($package, $repository, $reference);