From 29099a3af28fcd67b14efec487e8d16be1133633 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 26 Jun 2025 20:22:10 +0200 Subject: [PATCH 01/15] remove code for comparing search to sco --- improve-search-results/.gitignore | 2 - improve-search-results/README.md | 41 ---- improve-search-results/app.pl | 217 -------------------- improve-search-results/cpanfile | 6 - improve-search-results/cpanfile.snapshot | 241 ----------------------- 5 files changed, 507 deletions(-) delete mode 100644 improve-search-results/.gitignore delete mode 100644 improve-search-results/README.md delete mode 100755 improve-search-results/app.pl delete mode 100644 improve-search-results/cpanfile delete mode 100644 improve-search-results/cpanfile.snapshot diff --git a/improve-search-results/.gitignore b/improve-search-results/.gitignore deleted file mode 100644 index 3374723e1..000000000 --- a/improve-search-results/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/local/ -# carton/local::lib diff --git a/improve-search-results/README.md b/improve-search-results/README.md deleted file mode 100644 index 142d17e2a..000000000 --- a/improve-search-results/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -# Search result comparison system - -## Why - -We want to improve MetaCPAN's search results, getting them at least as good as search.cpan.org's but ideally -even better. - -## How - -Run multiple searches (via the API that the web UI now uses), with different weights (that are now arguments) and compare to each other (so one weighthing doesn't -then break another, or at least we can come to some -balance). - -### Installing - -You will need postgres installed with a database -that matches the current user and the current user needs -access (the MetaCPAN developer vm sets this up for you). - -```sh -cpanm Carton -carton install -``` - -### Running tests - -```sh - -carton exec /opt/perl-5.22.2/bin/perl ./app.pl eval 'app->perform_all_searches' -``` - -### Viewing results site -```sh -carton exec /opt/perl-5.22.2/bin/perl ./app.pl daemon -m production -l http://*:5000 -``` - - - - - diff --git a/improve-search-results/app.pl b/improve-search-results/app.pl deleted file mode 100755 index 6567c6fba..000000000 --- a/improve-search-results/app.pl +++ /dev/null @@ -1,217 +0,0 @@ -use Mojolicious::Lite; - -use Mojo::Pg; -use List::Util qw( first ); - -my $user = getpwuid($<); # for vagrant user on dev box - -# carton exec /opt/perl-5.22.2/bin/perl ./app.pl daemon -m production -l http://*:5000 - -helper pg => sub { state $pg = Mojo::Pg->new("postgresql:///${user}") }; - -app->pg->auto_migrate(1)->migrations->from_data; - -helper insert_search => sub { - my ( $c, $search, $expect ) = @_; - return !!$c->pg->db->query( <<' SQL', $search, $expect )->rows; - insert into searches (search, expect) values (?, ?) - SQL -}; - -helper insert_source => sub { - my ( $c, $name, $query ) = @_; - return !!$c->pg->db->query( <<' SQL', $name, $query )->rows; - insert into sources (name, query) values (?, ?, ?) - SQL -}; - -helper get_results => sub { - my $c = shift; - return $c->pg->db->query(<<' SQL')->expand->hash->{results}; - select json_object_agg(search, results) as results - from ( - select - searches.search, - json_object_agg(sources.name, results.rank) as results - from results - inner join searches on searches.id = results.search_id - inner join sources on sources.id = results.source_id - group by searches.search - ) x - SQL -}; - -helper perform_all_searches => sub { - my ($c) = @_; - my $queries = $c->pg->db->query(<<' SQL'); - select - searches.id as search_id, - sources.id as source_id, - searches.search, - searches.expect, - sources.name, - results.rank - from searches - cross join sources - left join results on searches.id = results.search_id - and sources.id = results.source_id - SQL - my $db = $c->pg->db; - my $sql - = 'insert into results (search_id, source_id, rank) values (?, ?, ?)'; - $queries->hashes->each( sub { - my $query = shift; - return if $query->{rank}; - my $rank = $c->perform_one_search( - @{$query}{qw/search expect name query/} ); - $db->query( $sql, @{$query}{qw/search_id source_id/}, $rank ); - } ); -}; - -helper perform_one_search => sub { - my ( $c, $search, $expect, $name, $query ) = @_; - - my $rank - = $name eq 'SCO' ? _perform_sco( $c, $search, $expect ) - : $name eq 'MWEB' ? _perform_mweb( $c, $search, $expect ) - : _perform_mquery( $c, $search, $expect, $query ); - - return $rank // 100; -}; - -sub _perform_sco { - my ( $c, $search, $expect ) = @_; - my $url = Mojo::URL->new('http://search.cpan.org/search?mode=all&n=100'); - $url->query( [ query => $search ] ); - my $tx = $c->app->ua->get($url); - my $res = $tx->res->dom->find('.sr')->map('all_text')->to_array; - my $idx = first { $res->[$_] eq $expect } @{ $res->to_array }; - return $idx < 0 ? undef : $idx + 1; -} - -sub _perform_mweb { - my ( $c, $search, $expect ) = @_; - my $url = Mojo::URL->new('https://metacpan.org/search?size=100'); - $url->query( [ q => $search ] ); - my $tx = $c->app->ua->get($url); - my $res - = $tx->res->dom->find('.module-result big strong a')->map('all_text') - ->to_array; - my $idx = first { $res->[$_] eq $expect } 0 .. $#{$res}; - return $idx < 0 ? undef : $idx + 1; -} - -sub _perform_mquery { } - -get '/' => 'index'; - -get '/results' => sub { - my $c = shift; - $c->render( json => $c->get_results ); -}; - -app->start; - -__DATA__ - -@@ index.html.ep - -<%== perform_all_searches %> - - - - - - MetaCPAN Search Comparison - - - - - - - - - - -@@ migrations - --- 1 up - -create table searches ( - id bigserial primary key, - search text not null unique, - expect text not null -); -insert into searches (search, expect) values -('tmpfile', 'File::Temp'), -('path', 'Path::Tiny'), -('dbix', 'DBIx::Class'), -('uri', 'URI'); - -create table sources ( - id bigserial primary key, - name text not null unique, - query text -); -insert into sources (name) values ('SCO'), ('MWEB'); - -create table results ( - id bigserial primary key, - search_id bigint references searches on delete cascade, - source_id bigint references sources on delete cascade, - rank integer -); - --- 1 down - -drop table if exists searches cascade; -drop table if exists sources cascade; -drop table if exists results cascade; diff --git a/improve-search-results/cpanfile b/improve-search-results/cpanfile deleted file mode 100644 index a4f1508a4..000000000 --- a/improve-search-results/cpanfile +++ /dev/null @@ -1,6 +0,0 @@ -requires 'perl', '5.010'; - -requires 'Mojolicious', 7.23; -requires 'Mojolicious::Lite', 0; -requires 'Mojo::Pg', 2.35; - diff --git a/improve-search-results/cpanfile.snapshot b/improve-search-results/cpanfile.snapshot deleted file mode 100644 index 2d4bd0476..000000000 --- a/improve-search-results/cpanfile.snapshot +++ /dev/null @@ -1,241 +0,0 @@ -# carton snapshot format: version 1.0 -DISTRIBUTIONS - DBD-Pg-3.5.3 - pathname: T/TU/TURNSTEP/DBD-Pg-3.5.3.tar.gz - provides: - Bundle::DBD::Pg v3.5.3 - DBD::Pg v3.5.3 - requirements: - DBI 1.614 - ExtUtils::MakeMaker 6.11 - Test::More 0.88 - Time::HiRes 0 - version 0 - DBI-1.636 - pathname: T/TI/TIMB/DBI-1.636.tar.gz - provides: - Bundle::DBI 12.008696 - DBD::DBM 0.08 - DBD::DBM::Statement 0.08 - DBD::DBM::Table 0.08 - DBD::DBM::db 0.08 - DBD::DBM::dr 0.08 - DBD::DBM::st 0.08 - DBD::ExampleP 12.014311 - DBD::ExampleP::db 12.014311 - DBD::ExampleP::dr 12.014311 - DBD::ExampleP::st 12.014311 - DBD::File 0.44 - DBD::File::DataSource::File 0.44 - DBD::File::DataSource::Stream 0.44 - DBD::File::Statement 0.44 - DBD::File::Table 0.44 - DBD::File::TableSource::FileSystem 0.44 - DBD::File::db 0.44 - DBD::File::dr 0.44 - DBD::File::st 0.44 - DBD::Gofer 0.015327 - DBD::Gofer::Policy::Base 0.010088 - DBD::Gofer::Policy::classic 0.010088 - DBD::Gofer::Policy::pedantic 0.010088 - DBD::Gofer::Policy::rush 0.010088 - DBD::Gofer::Transport::Base 0.014121 - DBD::Gofer::Transport::corostream undef - DBD::Gofer::Transport::null 0.010088 - DBD::Gofer::Transport::pipeone 0.010088 - DBD::Gofer::Transport::stream 0.014599 - DBD::Gofer::db 0.015327 - DBD::Gofer::dr 0.015327 - DBD::Gofer::st 0.015327 - DBD::NullP 12.014715 - DBD::NullP::db 12.014715 - DBD::NullP::dr 12.014715 - DBD::NullP::st 12.014715 - DBD::Proxy 0.2004 - DBD::Proxy::RPC::PlClient 0.2004 - DBD::Proxy::db 0.2004 - DBD::Proxy::dr 0.2004 - DBD::Proxy::st 0.2004 - DBD::Sponge 12.010003 - DBD::Sponge::db 12.010003 - DBD::Sponge::dr 12.010003 - DBD::Sponge::st 12.010003 - DBDI 12.015129 - DBI 1.636 - DBI::Const::GetInfo::ANSI 2.008697 - DBI::Const::GetInfo::ODBC 2.011374 - DBI::Const::GetInfoReturn 2.008697 - DBI::Const::GetInfoType 2.008697 - DBI::DBD 12.015129 - DBI::DBD::Metadata 2.014214 - DBI::DBD::SqlEngine 0.06 - DBI::DBD::SqlEngine::DataSource 0.06 - DBI::DBD::SqlEngine::Statement 0.06 - DBI::DBD::SqlEngine::Table 0.06 - DBI::DBD::SqlEngine::TableSource 0.06 - DBI::DBD::SqlEngine::TieMeta 0.06 - DBI::DBD::SqlEngine::TieTables 0.06 - DBI::DBD::SqlEngine::db 0.06 - DBI::DBD::SqlEngine::dr 0.06 - DBI::DBD::SqlEngine::st 0.06 - DBI::Gofer::Execute 0.014283 - DBI::Gofer::Request 0.012537 - DBI::Gofer::Response 0.011566 - DBI::Gofer::Serializer::Base 0.009950 - DBI::Gofer::Serializer::DataDumper 0.009950 - DBI::Gofer::Serializer::Storable 0.015586 - DBI::Gofer::Transport::Base 0.012537 - DBI::Gofer::Transport::pipeone 0.012537 - DBI::Gofer::Transport::stream 0.012537 - DBI::Profile 2.015065 - DBI::ProfileData 2.010008 - DBI::ProfileDumper 2.015325 - DBI::ProfileDumper::Apache 2.014121 - DBI::ProfileSubs 0.009396 - DBI::ProxyServer 0.3005 - DBI::ProxyServer::db 0.3005 - DBI::ProxyServer::dr 0.3005 - DBI::ProxyServer::st 0.3005 - DBI::SQL::Nano 1.015544 - DBI::SQL::Nano::Statement_ 1.015544 - DBI::SQL::Nano::Table_ 1.015544 - DBI::Util::CacheMemory 0.010315 - DBI::Util::_accessor 0.009479 - DBI::common 1.636 - requirements: - ExtUtils::MakeMaker 6.48 - Test::Simple 0.90 - perl 5.008 - Mojo-Pg-2.35 - pathname: S/SR/SRI/Mojo-Pg-2.35.tar.gz - provides: - Mojo::Pg 2.35 - Mojo::Pg::Database undef - Mojo::Pg::Migrations undef - Mojo::Pg::PubSub undef - Mojo::Pg::Results undef - Mojo::Pg::Transaction undef - requirements: - DBD::Pg 3.005001 - ExtUtils::MakeMaker 0 - Mojolicious 7.15 - perl 5.010001 - Mojolicious-7.23 - pathname: S/SR/SRI/Mojolicious-7.23.tar.gz - provides: - Mojo undef - Mojo::Asset undef - Mojo::Asset::File undef - Mojo::Asset::Memory undef - Mojo::Base undef - Mojo::ByteStream undef - Mojo::Cache undef - Mojo::Collection undef - Mojo::Content undef - Mojo::Content::MultiPart undef - Mojo::Content::Single undef - Mojo::Cookie undef - Mojo::Cookie::Request undef - Mojo::Cookie::Response undef - Mojo::DOM undef - Mojo::DOM::CSS undef - Mojo::DOM::HTML undef - Mojo::Date undef - Mojo::EventEmitter undef - Mojo::Exception undef - Mojo::File undef - Mojo::Headers undef - Mojo::HelloWorld undef - Mojo::Home undef - Mojo::IOLoop undef - Mojo::IOLoop::Client undef - Mojo::IOLoop::Delay undef - Mojo::IOLoop::Server undef - Mojo::IOLoop::Stream undef - Mojo::IOLoop::Subprocess undef - Mojo::IOLoop::TLS undef - Mojo::JSON undef - Mojo::JSON::Pointer undef - Mojo::Loader undef - Mojo::Log undef - Mojo::Message undef - Mojo::Message::Request undef - Mojo::Message::Response undef - Mojo::Parameters undef - Mojo::Path undef - Mojo::Reactor undef - Mojo::Reactor::EV undef - Mojo::Reactor::Poll undef - Mojo::Server undef - Mojo::Server::CGI undef - Mojo::Server::Daemon undef - Mojo::Server::Hypnotoad undef - Mojo::Server::Morbo undef - Mojo::Server::PSGI undef - Mojo::Server::PSGI::_IO undef - Mojo::Server::Prefork undef - Mojo::Template undef - Mojo::Transaction undef - Mojo::Transaction::HTTP undef - Mojo::Transaction::WebSocket undef - Mojo::URL undef - Mojo::Upload undef - Mojo::UserAgent undef - Mojo::UserAgent::CookieJar undef - Mojo::UserAgent::Proxy undef - Mojo::UserAgent::Server undef - Mojo::UserAgent::Transactor undef - Mojo::Util undef - Mojo::WebSocket undef - Mojolicious 7.23 - Mojolicious::Command undef - Mojolicious::Command::cgi undef - Mojolicious::Command::cpanify undef - Mojolicious::Command::daemon undef - Mojolicious::Command::eval undef - Mojolicious::Command::generate undef - Mojolicious::Command::generate::app undef - Mojolicious::Command::generate::lite_app undef - Mojolicious::Command::generate::makefile undef - Mojolicious::Command::generate::plugin undef - Mojolicious::Command::get undef - Mojolicious::Command::inflate undef - Mojolicious::Command::prefork undef - Mojolicious::Command::psgi undef - Mojolicious::Command::routes undef - Mojolicious::Command::test undef - Mojolicious::Command::version undef - Mojolicious::Commands undef - Mojolicious::Controller undef - Mojolicious::Lite undef - Mojolicious::Plugin undef - Mojolicious::Plugin::Config undef - Mojolicious::Plugin::Config::Sandbox undef - Mojolicious::Plugin::DefaultHelpers undef - Mojolicious::Plugin::EPLRenderer undef - Mojolicious::Plugin::EPRenderer undef - Mojolicious::Plugin::HeaderCondition undef - Mojolicious::Plugin::JSONConfig undef - Mojolicious::Plugin::Mount undef - Mojolicious::Plugin::PODRenderer undef - Mojolicious::Plugin::TagHelpers undef - Mojolicious::Plugins undef - Mojolicious::Renderer undef - Mojolicious::Routes undef - Mojolicious::Routes::Match undef - Mojolicious::Routes::Pattern undef - Mojolicious::Routes::Route undef - Mojolicious::Sessions undef - Mojolicious::Static undef - Mojolicious::Types undef - Mojolicious::Validator undef - Mojolicious::Validator::Validation undef - Test::Mojo undef - ojo undef - requirements: - ExtUtils::MakeMaker 0 - IO::Socket::IP 0.37 - JSON::PP 2.27103 - Pod::Simple 3.09 - Time::Local 1.2 - perl 5.010001 From cb69230d32a8d6225698131e40142e72121eedd0 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 26 Jun 2025 20:22:51 +0200 Subject: [PATCH 02/15] remove old docker deploy scripts --- deploy/build.sh | 14 -------------- deploy/push.sh | 11 ----------- deploy/vars.sh | 13 ------------- 3 files changed, 38 deletions(-) delete mode 100755 deploy/build.sh delete mode 100755 deploy/push.sh delete mode 100755 deploy/vars.sh diff --git a/deploy/build.sh b/deploy/build.sh deleted file mode 100755 index 64013f312..000000000 --- a/deploy/build.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -DEPLOY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -source "${DEPLOY_DIR}/vars.sh" - -# ## Go to where the docker file is -cd "${DEPLOY_DIR}/.." - -## Pull the latest docker file from docker hub if there is one -docker pull "$DOCKER_HUB_NAME" || true - -## Issue the build command, adding tags (from CONFIG.sh) -docker build --pull --cache-from "$DOCKER_HUB_NAME" --tag $DOCKER_HUB_NAME --tag $VERSION_TAG . diff --git a/deploy/push.sh b/deploy/push.sh deleted file mode 100755 index bb24bd2ff..000000000 --- a/deploy/push.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DEPLOY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -source "${DEPLOY_DIR}/vars.sh" - -cd "${DEPLOY_DIR}/.." - -docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWD" - -docker push "$DOCKER_HUB_NAME" \ No newline at end of file diff --git a/deploy/vars.sh b/deploy/vars.sh deleted file mode 100755 index 7431405a6..000000000 --- a/deploy/vars.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -## Edit this -if [ -z $DOCKER_IMAGE_NAME ]; then - echo "DOCKER_IMAGE_NAME is not defined" - exit 1; -fi - -## Should not need to edit this -export DOCKER_HUB_NAME="metacpan/${DOCKER_IMAGE_NAME}" -export VERSION="${TRAVIS_BUILD_NUMBER:-UNKNOWN-BUILD-NUMBER}" -export VERSION_TAG="${DOCKER_HUB_NAME}:${VERSION}" - From 84d173b9235bba1a362b3b4185fab44113e28cab Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 26 Jun 2025 20:28:26 +0200 Subject: [PATCH 03/15] separate cpanfile into correct phases --- cpanfile | 54 +++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/cpanfile b/cpanfile index ad67e1179..a6c462cb1 100644 --- a/cpanfile +++ b/cpanfile @@ -135,30 +135,34 @@ requires 'XML::XPath'; requires 'YAML::XS', '0.83'; # Mojolicious::Plugin::OpenAPI YAML loading # test requirements -requires 'Code::TidyAll', '0.82'; -requires 'Code::TidyAll::Plugin::UniqueLines'; -requires 'CPAN::Faker', '0.011'; -requires 'Devel::Confess'; -requires 'HTTP::Cookies', '6.10'; -requires 'MetaCPAN::Client', '2.029000'; -requires 'Module::Faker', '== 0.017'; -requires 'Module::Faker::Dist', '== 0.017'; -requires 'OrePAN2', '0.48'; -requires 'Parallel::ForkManager' => '2.02'; -requires 'Perl::Critic', '0.140'; -requires 'Perl::Tidy' => '== 20240511'; -requires 'PPI', '1.274'; # Perl::Critic -requires 'PPIx::QuoteLike', '0.022'; # Perl::Critic -requires 'PPIx::Regexp', '0.085'; # Perl::Critic -requires 'String::Format', '1.18'; # Perl::Critic -requires 'Test::Deep'; -requires 'Test::Fatal'; -requires 'Test::Harness', '3.44'; # Contains App::Prove -requires 'Test::More', '1.302190'; -requires 'Test::Perl::Critic', '1.04'; -requires 'Test::RequiresInternet'; -requires 'Test::Routine', '0.012'; -requires 'Test::Vars', '0.015'; +on test => sub { + requires 'Code::TidyAll', '0.82'; + requires 'Code::TidyAll::Plugin::UniqueLines'; + requires 'CPAN::Faker', '0.011'; + requires 'Devel::Confess'; + requires 'HTTP::Cookies', '6.10'; + requires 'MetaCPAN::Client', '2.029000'; + requires 'Module::Faker', '== 0.017'; + requires 'Module::Faker::Dist', '== 0.017'; + requires 'OrePAN2', '0.48'; + requires 'Test::Deep'; + requires 'Test::Fatal'; + requires 'Test::Harness', '3.44'; # Contains App::Prove + requires 'Test::More', '1.302190'; + requires 'Test::RequiresInternet'; + requires 'Test::Routine', '0.012'; + requires 'Test::Vars', '0.015'; +}; # author requirements -requires 'App::perlimports'; +on develop => sub { + requires 'App::perlimports'; + requires 'Code::TidyAll::Plugin::UniqueLines'; + requires 'Code::TidyAll', '0.82'; + requires 'Perl::Critic', '0.140'; + requires 'Perl::Tidy' => '== 20240511'; + requires 'PPI', '1.274'; # Perl::Critic + requires 'PPIx::QuoteLike', '0.022'; # Perl::Critic + requires 'PPIx::Regexp', '0.085'; # Perl::Critic + requires 'String::Format', '1.18'; # Perl::Critic +}; From 681267065c374e24e30a1386813b3936e0d76a11 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:21:43 +0200 Subject: [PATCH 04/15] install Devel::Cover as part of developer tools --- cpanfile | 1 + 1 file changed, 1 insertion(+) diff --git a/cpanfile b/cpanfile index a6c462cb1..79beb1a2b 100644 --- a/cpanfile +++ b/cpanfile @@ -165,4 +165,5 @@ on develop => sub { requires 'PPIx::QuoteLike', '0.022'; # Perl::Critic requires 'PPIx::Regexp', '0.085'; # Perl::Critic requires 'String::Format', '1.18'; # Perl::Critic + requires 'Devel::Cover'; }; From 9b9c4056b99fd73ee9112ab069221b30f4090575 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 09:30:00 +0200 Subject: [PATCH 05/15] use correct json module when showing es info --- lib/MetaCPAN/Script/Mapping.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index 801b05a39..8fabfb4b8 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -162,7 +162,7 @@ sub show_info { 'cluster_info' => \%{ $self->cluster_info }, 'indices_info' => \%{ $self->indices_info }, }; - log_info { JSON->new->utf8->pretty->encode($info_rs) }; + log_info { Cpanel::JSON::XS->new->utf8->pretty->encode($info_rs) }; } sub _build_index_config { From 635af4c8d7fdf899558eb0d0f1dba194157ddc92 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 09:30:27 +0200 Subject: [PATCH 06/15] perlimports --- lib/MetaCPAN/Model/ESWrapper.pm | 2 +- lib/MetaCPAN/Role/Logger.pm | 2 +- lib/MetaCPAN/Script/Mapping.pm | 5 ++--- lib/MetaCPAN/Types/Internal.pm | 10 ++++++---- t/lib/MetaCPAN/TestServer.pm | 1 - t/server/controller/author.t | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/MetaCPAN/Model/ESWrapper.pm b/lib/MetaCPAN/Model/ESWrapper.pm index 9cc17b497..cd990c897 100644 --- a/lib/MetaCPAN/Model/ESWrapper.pm +++ b/lib/MetaCPAN/Model/ESWrapper.pm @@ -2,7 +2,7 @@ package MetaCPAN::Model::ESWrapper; use strict; use warnings; -use MetaCPAN::Types::TypeTiny qw(ES); +use MetaCPAN::Types::TypeTiny qw( ES ); sub new { my ( $class, $es ) = @_; diff --git a/lib/MetaCPAN/Role/Logger.pm b/lib/MetaCPAN/Role/Logger.pm index 258ccfa0a..95d80d662 100644 --- a/lib/MetaCPAN/Role/Logger.pm +++ b/lib/MetaCPAN/Role/Logger.pm @@ -6,7 +6,7 @@ use Moose::Role; use Log::Contextual qw( set_logger ); use Log::Log4perl ':easy'; use MetaCPAN::Types::TypeTiny qw( Logger Str ); -use MooseX::Getopt (); +use MooseX::Getopt (); ## no perlimports use Path::Tiny qw( path ); has level => ( diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index 8fabfb4b8..128c78b76 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -2,11 +2,10 @@ package MetaCPAN::Script::Mapping; use Moose; -use Cpanel::JSON::XS qw( decode_json ); -use DateTime (); +use Cpanel::JSON::XS (); use Log::Contextual qw( :log ); use MetaCPAN::ESConfig qw( es_config ); -use MetaCPAN::Types::TypeTiny qw( Bool HashRef Int Str ); +use MetaCPAN::Types::TypeTiny qw( Bool HashRef Int ); use Time::HiRes qw( sleep time ); use constant { diff --git a/lib/MetaCPAN/Types/Internal.pm b/lib/MetaCPAN/Types/Internal.pm index 3bffb47f7..b991f8fe3 100644 --- a/lib/MetaCPAN/Types/Internal.pm +++ b/lib/MetaCPAN/Types/Internal.pm @@ -7,7 +7,7 @@ use ElasticSearchX::Model::Document::Mapping (); use ElasticSearchX::Model::Document::Types qw( Type ); use MetaCPAN::Util qw( is_bool true false ); use MooseX::Getopt::OptionTypeMap (); -use MooseX::Types::Moose qw( Item Any Bool ArrayRef HashRef ); +use MooseX::Types::Moose qw( ArrayRef Bool HashRef Item ); use MooseX::Types -declare => [ qw( ESBool @@ -24,7 +24,7 @@ coerce Module, from ArrayRef, via { @$_ ]; }; coerce Module, from HashRef, via { - require MetaCPAN::Document::Module; + require MetaCPAN::Document::Module; ## no perlimports [ MetaCPAN::Document::Module->new($_) ]; }; @@ -40,7 +40,7 @@ coerce Identity, from ArrayRef, via { ]; }; coerce Identity, from HashRef, via { - require MetaCPAN::Model::User::Identity; + require MetaCPAN::Model::User::Identity; ## no perlimports [ MetaCPAN::Model::User::Identity->new($_) ]; }; @@ -56,7 +56,7 @@ coerce Dependency, from ArrayRef, via { ]; }; coerce Dependency, from HashRef, via { - require MetaCPAN::Document::Dependency; + require MetaCPAN::Document::Dependency; ## no perlimports [ MetaCPAN::Document::Dependency->new($_) ]; }; @@ -72,7 +72,9 @@ coerce Profile, from ArrayRef, via { ]; }; coerce Profile, from HashRef, via { + ## no perlimports require MetaCPAN::Document::Author::Profile; + ## use perlimports [ MetaCPAN::Document::Author::Profile->new($_) ]; }; diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index f37309960..da329b7a1 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -2,7 +2,6 @@ package MetaCPAN::TestServer; use MetaCPAN::Moose; -use Cpanel::JSON::XS qw( encode_json ); use MetaCPAN::ESConfig qw( es_config ); use MetaCPAN::Script::Author (); use MetaCPAN::Script::Cover (); diff --git a/t/server/controller/author.t b/t/server/controller/author.t index 7b5538ad0..5abe58e00 100644 --- a/t/server/controller/author.t +++ b/t/server/controller/author.t @@ -2,7 +2,7 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( app GET POST test_psgi es ); +use MetaCPAN::Server::Test qw( app es GET POST test_psgi ); use MetaCPAN::TestHelpers qw( decode_json_ok test_cache_headers ); use Test::More; From fe4e5f86870e717703ffdd8d6e9fa558fdd501c5 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 26 Jun 2025 20:33:46 +0200 Subject: [PATCH 07/15] switch from tidyall to precious --- .github/workflows/code-formatting.yml | 52 +++++++++++++++++++++++++++ .gitignore | 3 ++ bin/install-precious | 24 +++++++++++++ cpanfile | 4 --- git/hooks/pre-commit | 28 ++++++++------- precious.toml | 6 ++-- t/fff_tidyall.t | 8 ----- tidyall.ini | 15 -------- 8 files changed, 97 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/code-formatting.yml create mode 100755 bin/install-precious delete mode 100644 t/fff_tidyall.t delete mode 100644 tidyall.ini diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml new file mode 100644 index 000000000..5eb31e5e0 --- /dev/null +++ b/.github/workflows/code-formatting.yml @@ -0,0 +1,52 @@ +--- +name: Code Formatting +on: + push: + branches: + - 'master' + merge_group: + pull_request: + branches: + - '*' + workflow_dispatch: + +jobs: + code-formatting: + runs-on: ubuntu-24.04 + name: Code Formatting + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Fetch base ref + if: ${{ github.event.pull_request }} + run: git fetch origin ${{ github.base_ref }}:upstream + - name: Install Carton + uses: perl-actions/install-with-cpm@v1 + with: + install: Carton + - name: Install CPAN deps + uses: perl-actions/install-with-cpm@v1 + with: + cpanfile: 'cpanfile' + args: > + --resolver=snapshot + --without-runtime + --without-test + --without-build + --with-develop + - name: Install precious + run: ./bin/install-precious /usr/local/bin + env: + GITHUB_TOKEN: ${{ github.token }} + - run: perltidy --version + - name: Select files + id: select-files + run: | + if [[ -n "${{ github.event.pull_request.number }}" ]]; then + echo 'precious-args=--git-diff-from upstream' >> "$GITHUB_OUTPUT" + else + echo 'precious-args=--all' >> "$GITHUB_OUTPUT" + fi + - name: Lint files + run: precious lint ${{ steps.select-files.outputs.precious-args }} diff --git a/.gitignore b/.gitignore index 194aaad50..cd6e79b24 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ /perltidy.LOG /pm_to_blib /var +/bin/omegasort +/bin/precious +/bin/ubi /etc/metacpan_local.pl /t/var/darkpan/ /t/var/log/ diff --git a/bin/install-precious b/bin/install-precious new file mode 100755 index 000000000..0d712a470 --- /dev/null +++ b/bin/install-precious @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This is for installing precious and other 3rd party libs needed for linting +# in CI + +set -euo pipefail + +if [ -z "${1:-}" ]; then + echo "usage: ./bin/install-precious /path/to/bin/dir" + exit 1 +fi + +TARGET=$1 +export TARGET + +TARGET=$1 +export TARGET + +curl --silent --location \ + https://raw.githubusercontent.com/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.sh | + sh + +ubi --project houseabsolute/omegasort --in "$TARGET" +ubi --project houseabsolute/precious --in "$TARGET" diff --git a/cpanfile b/cpanfile index 79beb1a2b..e647fcde6 100644 --- a/cpanfile +++ b/cpanfile @@ -136,8 +136,6 @@ requires 'YAML::XS', '0.83'; # Mojolicious::Plugin::OpenAPI YAML loading # test requirements on test => sub { - requires 'Code::TidyAll', '0.82'; - requires 'Code::TidyAll::Plugin::UniqueLines'; requires 'CPAN::Faker', '0.011'; requires 'Devel::Confess'; requires 'HTTP::Cookies', '6.10'; @@ -157,8 +155,6 @@ on test => sub { # author requirements on develop => sub { requires 'App::perlimports'; - requires 'Code::TidyAll::Plugin::UniqueLines'; - requires 'Code::TidyAll', '0.82'; requires 'Perl::Critic', '0.140'; requires 'Perl::Tidy' => '== 20240511'; requires 'PPI', '1.274'; # Perl::Critic diff --git a/git/hooks/pre-commit b/git/hooks/pre-commit index 7b1572230..9256ae257 100755 --- a/git/hooks/pre-commit +++ b/git/hooks/pre-commit @@ -1,13 +1,15 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use Config; -# Hack to use carton's local::lib. -use lib 'local/lib/perl5'; -$ENV{PATH} .= $Config{path_sep}.'local/bin'; -$ENV{PERL5LIB} = join $Config{path_sep}, - grep defined, $ENV{PERL5LIB}, 'local/lib/perl5'; - -use Code::TidyAll::Git::Precommit; -Code::TidyAll::Git::Precommit->check( no_stash => 1 ); +#!/bin/bash + +declare -i status +status=0 + +PRECIOUS=$(which precious) +if [[ -z $PRECIOUS ]]; then + PRECIOUS=./bin/precious +fi + +if ! "$PRECIOUS" lint -s; then + status+=1 +fi + +exit $status diff --git a/precious.toml b/precious.toml index 54034f059..21be953fd 100644 --- a/precious.toml +++ b/precious.toml @@ -6,7 +6,7 @@ excludes = [ [commands.perlimports] type = "both" -include = [ "**/*.{pl,pm,t,psgi}" ] +include = [ "**/*.{pl,pm,t,psgi}", "bin/metacpan" ] cmd = [ "perlimports" ] lint-flags = ["--lint" ] tidy-flags = ["-i" ] @@ -15,14 +15,14 @@ expect-stderr = true [commands.perlcritic] type = "lint" -include = [ "**/*.{pl,pm,t,psgi}" ] +include = [ "**/*.{pl,pm,t,psgi}", "bin/metacpan" ] cmd = [ "perlcritic", "--profile=$PRECIOUS_ROOT/.perlcriticrc" ] ok-exit-codes = 0 lint-failure-exit-codes = 2 [commands.perltidy] type = "both" -include = [ "**/*.{pl,pm,t,psgi}" ] +include = [ "**/*.{pl,pm,t,psgi}", "bin/metacpan" ] cmd = [ "perltidy", "--profile=$PRECIOUS_ROOT/.perltidyrc" ] lint-flags = [ "--assert-tidy", "--no-standard-output", "--outfile=/dev/null" ] tidy-flags = [ "--backup-and-modify-in-place", "--backup-file-extension=/" ] diff --git a/t/fff_tidyall.t b/t/fff_tidyall.t deleted file mode 100644 index e96e8e278..000000000 --- a/t/fff_tidyall.t +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use lib 't/lib'; - -use Test::Code::TidyAll qw( tidyall_ok ); -tidyall_ok( verbose => $ENV{TEST_VERBOSE} ); diff --git a/tidyall.ini b/tidyall.ini deleted file mode 100644 index b29f471e9..000000000 --- a/tidyall.ini +++ /dev/null @@ -1,15 +0,0 @@ -jobs = 5 - -[PerlTidy] -select = {bin,lib,t,xt}/**/*.{pl,pm,t,psgi} -select = bin/{metacpan,wait-for-open} -select = app.psgi -ignore = t/var/**/* -argv = --profile=$ROOT/.perltidyrc - -[PerlCritic] -select = {bin,lib,t,xt}/**/*.{pl,pm,t,psgi} -select = bin/{metacpan,wait-for-open} - -[UniqueLines] -select = .gitignore From 8e440777b34077544d19bc516b49ce64e48c65e7 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 26 Jun 2025 21:36:16 +0200 Subject: [PATCH 08/15] add simple healthcheck end point --- lib/MetaCPAN/Server/Controller/Root.pm | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Controller/Root.pm b/lib/MetaCPAN/Server/Controller/Root.pm index d0fb311bf..a7f0d4fe9 100644 --- a/lib/MetaCPAN/Server/Controller/Root.pm +++ b/lib/MetaCPAN/Server/Controller/Root.pm @@ -4,6 +4,7 @@ use strict; use warnings; use Moose; +use MetaCPAN::Util qw( true ); BEGIN { extends 'MetaCPAN::Server::Controller' } @@ -68,6 +69,12 @@ sub robots : Path("robots.txt") Args(0) { $c->res->body("User-agent: *\nDisallow: /\n"); } +sub healthcheck : Local Args(0) { + my ( $self, $c ) = @_; + $c->stash( { success => true } ); + $c->forward( $c->view('JSON') ); +} + sub end : ActionClass('RenderView') { my ( $self, $c ) = @_; if ( $c->controller->does('MetaCPAN::Server::Role::JSONP') @@ -99,7 +106,6 @@ sub end : ActionClass('RenderView') { # cdn cache time $c->cdn_never_cache(1); } - } 1; From b082aa98b7ba4d703d7bf80464e5b7c26f0d4d17 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 08:55:39 +0200 Subject: [PATCH 09/15] refresh Dockerfile for better caching Update the Dockerfile to better use caching by separating the prereq building. Also switch the server to uwsgi rather than using the Mojolicious wrapper. It provided little benefit and caused numerous problems, and we haven't been using it in production for a while. --- Dockerfile | 117 +++++++++++++++++++++++++++++++++++++-------- docker-compose.yml | 17 +++++++ 2 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 976a9f5f2..2948c9f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,101 @@ -# hadolint ignore=DL3007 -FROM metacpan/metacpan-base:latest - -COPY cpanfile cpanfile.snapshot /metacpan-api/ -WORKDIR /metacpan-api - -# CPM installations of dependencies does not install or run tests. This is -# because the modules themselves have been tested, and the metacpan use of the -# modules is tested by the test suite. Removing the tests, reduces the overall -# size of the images. -RUN mkdir /CPAN \ - && apt-get update \ - && apt-get satisfy -y --no-install-recommends 'rsync (>= 3.2.3)' 'jq (>= 1.6)' \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && cpm install --global \ - && git config --global --add safe.directory /metacpan-api \ - && rm -fr /root/.cpanm /root/.perl-cpm /var/cache/apt/lists/* /tmp/* +ARG SLIM_BUILD +ARG MAYBE_BASE_BUILD=${SLIM_BUILD:+server-base-slim} +ARG BASE_BUILD=${MAYBE_BASE_BUILD:-server-base} + +################### Web Server Base +FROM metacpan/metacpan-base:main-20250531-090128 AS server-base +FROM metacpan/metacpan-base:main-20250531-090129-slim AS server-base-slim + +################### CPAN Prereqs +FROM server-base AS build-cpan-prereqs +SHELL [ "/bin/bash", "-euo", "pipefail", "-c" ] + +WORKDIR /app/ + +COPY cpanfile cpanfile.snapshot ./ +RUN \ + --mount=type=cache,target=/root/.perl-cpm,sharing=private \ +< Date: Sat, 28 Jun 2025 08:36:59 +0200 Subject: [PATCH 10/15] remove Mojolicious web code The main applications is in Catalyst, and using MountPSGI continued to cause problems. Our deployments are moving to uwsgi. Unless the entire app is rewritten in Mojolicious, this is just extraneous code. --- lib/MetaCPAN/API.pm | 87 +------------ lib/MetaCPAN/API/Controller/Cover.pm | 16 --- lib/MetaCPAN/API/Controller/Queue.pm | 19 --- lib/MetaCPAN/API/Controller/Search.pm | 32 ----- lib/MetaCPAN/API/Model/Cover.pm | 33 ----- lib/MetaCPAN/API/Model/Download.pm | 10 -- root/static/definitions/common.yml | 13 -- root/static/definitions/definitions.yml | 5 - root/static/definitions/results.yml | 80 ------------ root/static/index.html | 24 ---- root/static/requests/cover.yml | 60 --------- root/static/requests/release.yml | 48 ------- root/static/requests/search.yml | 124 ------------------- root/static/v1.yml | 26 ---- t/api/controller/admin.t | 39 ------ t/api/controller/cover.t | 56 --------- t/api/controller/search/first.t | 14 --- t/api/controller/search/web.t | 73 ----------- templates/admin/identity_search_form.html.ep | 13 -- templates/admin/index.html.ep | 8 -- templates/admin/search_identities.html.ep | 3 - templates/layouts/default.html.ep | 5 - templates/queue/index_release.html.ep | 0 23 files changed, 6 insertions(+), 782 deletions(-) delete mode 100644 lib/MetaCPAN/API/Controller/Cover.pm delete mode 100644 lib/MetaCPAN/API/Controller/Queue.pm delete mode 100644 lib/MetaCPAN/API/Controller/Search.pm delete mode 100644 lib/MetaCPAN/API/Model/Cover.pm delete mode 100644 lib/MetaCPAN/API/Model/Download.pm delete mode 100644 root/static/definitions/common.yml delete mode 100644 root/static/definitions/definitions.yml delete mode 100644 root/static/definitions/results.yml delete mode 100644 root/static/index.html delete mode 100644 root/static/requests/cover.yml delete mode 100644 root/static/requests/release.yml delete mode 100644 root/static/requests/search.yml delete mode 100644 root/static/v1.yml delete mode 100644 t/api/controller/admin.t delete mode 100644 t/api/controller/cover.t delete mode 100644 t/api/controller/search/first.t delete mode 100644 t/api/controller/search/web.t delete mode 100644 templates/admin/identity_search_form.html.ep delete mode 100644 templates/admin/index.html.ep delete mode 100644 templates/admin/search_identities.html.ep delete mode 100644 templates/layouts/default.html.ep delete mode 100644 templates/queue/index_release.html.ep diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index ab7a9f66b..86b29ab84 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -2,34 +2,20 @@ package MetaCPAN::API; =head1 DESCRIPTION -This is the API web server interface. - - # On vagrant VM - ./bin/run morbo bin/api.pl +This is the API Minion server. # Display information on jobs in queue ./bin/run bin/api.pl minion job -s -To run the api web server, run the following on one of the servers: - - # Run the daemon on a local port (tunnel to display on your browser) - ./bin/run bin/api.pl daemon - =cut use Mojo::Base 'Mojolicious'; -use File::Temp (); -use List::Util qw( any ); -use MetaCPAN::Script::Runner (); -use Try::Tiny qw( catch try ); -use MetaCPAN::Server::Config (); -use MetaCPAN::Types::TypeTiny qw( ES ); - -has es => sub { - ES->assert_coerce( - MetaCPAN::Server::Config::config()->{elasticsearch_servers} ); -}; +use File::Temp (); +use List::Util qw( any ); +use MetaCPAN::Script::Runner (); +use Try::Tiny qw( catch try ); +use MetaCPAN::Server::Config (); sub startup { my $self = shift; @@ -55,7 +41,6 @@ sub startup { $self->minion->add_task( index_favorite => $self->_gen_index_task_sub('favorite') ); - $self->plugin('MetaCPAN::API::Plugin::Model'); $self->_set_up_routes; } @@ -116,36 +101,7 @@ sub _set_up_routes { ); $self->_set_up_oauth_routes; - - $admin->get('home')->to('admin#home')->name('admin-home'); - $admin->post('enqueue')->to('queue#enqueue')->name('enqueue'); - $admin->post('search-identities')->to('admin#search_identities') - ->name('search-identities'); - $admin->get('index-release')->to('queue#index_release') - ->name('index-release'); - $admin->get('identity-search-form')->to('admin#identity_search_form') - ->name('identity_search_form'); - $self->plugin( 'Minion::Admin' => { route => $admin->any('/minion') } ); - $self->plugin( - 'OpenAPI' => { url => $self->home->rel_file('root/static/v1.yml') } ); - -# This route is for when nginx gets updated to no longer strip the `/v1` path. -# By retaining the `/v1` path the OpenAPI spec is picked up and passed -# through Mojolicous. The `rewrite` parameter is stripping the `/v1` before -# it is passed to Catalyst allowing the previous logic to be followed. - $self->plugin( - MountPSGI => { - '/v1' => $self->home->child('app.psgi')->to_string, - rewrite => 1 - } - ); - -# XXX Catch cases when `v1` has been stripped by nginx until migration is complete -# XXX then this path can be removed. - $self->plugin( - MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); - } sub _is_admin { @@ -205,43 +161,12 @@ sub _set_up_oauth_routes { $c->session( is_logged_in => 1 ); $c->session( github_username => $username ); if ( $self->_is_admin($username) ) { - $c->session( github_username => $username ); $c->redirect_to('/admin'); return; } $c->redirect_to( $self->config->{front_end_url} ); }, ); - - $self->plugin( - 'Web::Auth', - module => 'Google', - key => $oauth->{google}->{key}, - secret => $oauth->{google}->{secret}, - user_info => 1, - on_finished => sub { - my ( $c, $access_token, $account_info ) = @_; - my $username = $account_info->{login}; - $c->session( is_logged_in => 1 ); - $c->session( google_username => $username ); - $c->redirect_to( $self->config->{front_end_url} ); - }, - ); - - $self->plugin( - 'Web::Auth', - module => 'Twitter', - key => $oauth->{twitter}->{key}, - secret => $oauth->{twitter}->{secret}, - user_info => 1, - on_finished => sub { - my ( $c, $access_token, $access_secret, $account_info ) = @_; - my $username = $account_info->{screen_name}; - $c->session( is_logged_in => 1 ); - $c->session( twitter_username => $username ); - $c->redirect_to( $self->config->{front_end_url} ); - }, - ); } 1; diff --git a/lib/MetaCPAN/API/Controller/Cover.pm b/lib/MetaCPAN/API/Controller/Cover.pm deleted file mode 100644 index 7479b5154..000000000 --- a/lib/MetaCPAN/API/Controller/Cover.pm +++ /dev/null @@ -1,16 +0,0 @@ -package MetaCPAN::API::Controller::Cover; - -use Mojo::Base 'Mojolicious::Controller'; - -sub lookup { - my $c = shift; - return unless $c->openapi->valid_input; - my $args = $c->validation->output; - - my $results = $c->model->cover->find_release_coverage( $args->{name} ); - return $c->render( openapi => $results ) if $results; - $c->rendered(404); -} - -1; - diff --git a/lib/MetaCPAN/API/Controller/Queue.pm b/lib/MetaCPAN/API/Controller/Queue.pm deleted file mode 100644 index d35c66167..000000000 --- a/lib/MetaCPAN/API/Controller/Queue.pm +++ /dev/null @@ -1,19 +0,0 @@ -package MetaCPAN::API::Controller::Queue; - -use Mojo::Base 'Mojolicious::Controller'; - -my $rel - = 'https://cpan.metacpan.org/authors/id/O/OA/OALDERS/HTML-Restrict-2.2.2.tar.gz'; - -sub enqueue { - my $self = shift; - $self->minion->enqueue( index_release => [ '--latest', $rel ] ); - $self->render( text => 'OK' ); -} - -sub index_release { - my $self = shift; - $self->render( text => 'ok' ); -} - -1; diff --git a/lib/MetaCPAN/API/Controller/Search.pm b/lib/MetaCPAN/API/Controller/Search.pm deleted file mode 100644 index fbc21bdcc..000000000 --- a/lib/MetaCPAN/API/Controller/Search.pm +++ /dev/null @@ -1,32 +0,0 @@ -package MetaCPAN::API::Controller::Search; - -use Mojo::Base 'Mojolicious::Controller'; - -sub first { - my $c = shift; - return unless $c->openapi->valid_input; - my $args = $c->validation->output; - - my $results = $c->model->search->search_for_first_result( $args->{q} ); - return $c->render( openapi => $results ) if $results; - $c->rendered(404); -} - -sub web { - my $c = shift; - return unless $c->openapi->valid_input; - my $args = $c->validation->output; - - my $query = $args->{q}; - my $size = $args->{page_size} // $args->{size} // 20; - my $page = $args->{page} // ( 1 + int( ( $args->{from} // 0 ) / $size ) ); - my $collapsed = $args->{collapsed}; - - my $results - = $c->model->search->search_web( $query, $page, $size, $collapsed ); - - return $c->render( json => $results ); -} - -1; - diff --git a/lib/MetaCPAN/API/Model/Cover.pm b/lib/MetaCPAN/API/Model/Cover.pm deleted file mode 100644 index 243ccae7d..000000000 --- a/lib/MetaCPAN/API/Model/Cover.pm +++ /dev/null @@ -1,33 +0,0 @@ -package MetaCPAN::API::Model::Cover; - -use MetaCPAN::ESConfig qw( es_doc_path ); -use MetaCPAN::Moose; - -use MetaCPAN::Util qw(hit_total); - -with 'MetaCPAN::API::Model::Role::ES'; - -sub find_release_coverage { - my ( $self, $release ) = @_; - - my $query = +{ term => { release => $release } }; - - my $res = $self->_run_query( - es_doc_path('cover'), - body => { - query => $query, - size => 999, - } - ); - hit_total($res) or return {}; - - return +{ - %{ $res->{hits}{hits}[0]{_source} }, - url => "http://cpancover.com/latest/$release/index.html", - }; -} - -__PACKAGE__->meta->make_immutable; - -1; - diff --git a/lib/MetaCPAN/API/Model/Download.pm b/lib/MetaCPAN/API/Model/Download.pm deleted file mode 100644 index 7900b8bf3..000000000 --- a/lib/MetaCPAN/API/Model/Download.pm +++ /dev/null @@ -1,10 +0,0 @@ -package MetaCPAN::API::Model::Download; - -use MetaCPAN::Moose; - -with 'MetaCPAN::API::Model::Role::ES'; - -__PACKAGE__->meta->make_immutable; - -1; - diff --git a/root/static/definitions/common.yml b/root/static/definitions/common.yml deleted file mode 100644 index 4835b3059..000000000 --- a/root/static/definitions/common.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -ErrorModel: - type: "object" - required: - - "code" - - "message" - properties: - code: - type: "integer" - format: "int32" - message: - type: "string" diff --git a/root/static/definitions/definitions.yml b/root/static/definitions/definitions.yml deleted file mode 100644 index b292922fa..000000000 --- a/root/static/definitions/definitions.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -# Maintain names and descriptions of common attributes -dist_fav_count: - description: Number of times favorited diff --git a/root/static/definitions/results.yml b/root/static/definitions/results.yml deleted file mode 100644 index eaf7183f1..000000000 --- a/root/static/definitions/results.yml +++ /dev/null @@ -1,80 +0,0 @@ ---- - -search_result_items: - type: object - properties: - distribution: - type: string - hits: - title: Hits - type: array - items: - $ref: "#/search_result_hit" - - total: - type: integer -search_result_hit: - type: object - properties: - description: - type: string - documentation: - type: string - authorized: - type: boolean - path: - type: string - author: - type: string - id: - type: string - date: - type: string - favorites: - type: - - "integer" - - "null" - status: - type: string - score: - type: number - module: - type: array - items: - type: object - properties: - associated_pod: - type: string - indexed: - type: boolean - name: - type: string - authorized: - type: boolean - version_numified: - type: number - distribution: - type: string - indexed: - type: boolean - pod_lines: - type: array - abstract: - type: string - release: - type: string -dependency: - type: object - properties: - module: - type: string - # "Mojolicious", - phase: - type: string - # "runtime", - version: - type: string - # "8", - relationship: - type: string - # "requires" diff --git a/root/static/index.html b/root/static/index.html deleted file mode 100644 index 3c119143f..000000000 --- a/root/static/index.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - MetaCPAN API - - - - - - - - - - - - - diff --git a/root/static/requests/cover.yml b/root/static/requests/cover.yml deleted file mode 100644 index 639e96730..000000000 --- a/root/static/requests/cover.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- - -cover: - get: - tags: - - Coverage - operationId: cover - x-mojo-to: Cover#lookup - summary: Get coverage details about a release - parameters: - - name: name - in: path - description: | - The name of the Release - type: string - required: true - # tell Mojolicious to use relaxed placeholders when - # parsing this parameter (dots are allowed) - x-mojo-placeholder: "#" - responses: - 200: - description: Release response - schema: - type: object - properties: - url: - type: string - description: URL for cpancover report - version: - type: string - description: Package version string - release: - type: string - description: Package name with version - criteria: - type: object - description: CPAN Cover results - properties: - total: - type: string - description: Percentage of total code coverage - condition: - type: string - description: Percentage of condition code coverage - statement: - type: string - description: Percentage of statement code coverage - branch: - type: string - description: Percentage of branch code coverage - subroutine: - type: string - description: Percentage of subroutine code coverage - distribution: - type: string - description: Name of the distribution - default: - description: "unexpected error" - schema: - $ref: "../definitions/common.yml#/ErrorModel" diff --git a/root/static/requests/release.yml b/root/static/requests/release.yml deleted file mode 100644 index a872d0364..000000000 --- a/root/static/requests/release.yml +++ /dev/null @@ -1,48 +0,0 @@ - /release/recent: - get: - tags: - - Release - operationId: release_recent - x-mojo-to: Release#recent - summary: Get recent releases - parameters: - - in: path - name: name - description: | - The name of the Release - type: string - required: true - responses: - 200: - description: Release response - schema: - type: object - properties: - name: - type: string - dependency: - type: array - items: - $ref: "./definitions/results.yml#/dependency" - /release/{name}: - get: - tags: - - Release - operationId: release_by_name - x-mojo-to: Release#by_name - summary: Get details about a release - parameters: - - in: path - name: name - description: | - The name of the Release - type: string - required: true - responses: - 200: - description: Release response - schema: - type: object - properties: - name: - type: string diff --git a/root/static/requests/search.yml b/root/static/requests/search.yml deleted file mode 100644 index 60f6824af..000000000 --- a/root/static/requests/search.yml +++ /dev/null @@ -1,124 +0,0 @@ ---- - -search_web: - get: - tags: - - Search - operationId: search_web - x-mojo-to: Search#web - summary: Perform API search in the same fashion as the Web UI - parameters: - - name: q - in: query - description: | - The query search term. If the search term contains a term with the - tags `dist:` or `module:` results will be in expanded form, otherwise - collapsed form. - - See also `collapsed` - type: string - required: true - - name: page - in: query - description: The page of the results to return - type: integer - - name: page_size - in: query - description: Number of results per page - type: integer - - name: from - in: query - description: | - The offset to use in the result set. Deprecated. Only used if `page` - is not set. - type: integer - default: 0 - - name: size - in: query - description: | - Number of results per page. Deprecated. Only used if `page_size` is - not set. - type: integer - default: 20 - - name: collapsed - in: query - description: | - Force a collapsed even when searching for a particular - distribution or module name. - type: boolean - responses: - 200: - description: Search response - schema: - type: object - properties: - total: - type: integer - took: - type: number - collapsed: - type: boolean - results: - title: Results - type: array - items: - $ref: "../definitions/results.yml#/search_result_items" -search_first: - get: - tags: - - Search - operationId: search_for_first - x-mojo-to: Search#first - summary: Perform API search and return the first result (I'm Feeling Lucky) - parameters: - - name: q - in: query - description: | - The query search term. - type: string - required: true - responses: - 200: - description: Search response - schema: - type: object - properties: - path: - type: string - description: Relative path to module with full name - authorized: - type: boolean - description: - type: string - description: Module description - id: - type: string - distribution: - type: string - description: Name of the distribution the module is contained in - author: - type: string - description: Module author ID - release: - type: string - description: Package name with version - status: - type: string - abstract.analyzed: - type: string - description: The module's abstract as analyzed from POD - dist_fav_count: - type: integer - description: Number of times favorited - date: - type: string - description: date module was indexed - documentation: - type: string - pod_lines: - type: array - items: - type: integer - indexed: - type: boolean - description: Is the module indexed by PAUSE diff --git a/root/static/v1.yml b/root/static/v1.yml deleted file mode 100644 index 59c0f3ff1..000000000 --- a/root/static/v1.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - -swagger: "2.0" -info: - version: "1.0.0" - title: "MetaCPAN API" -basePath: "/v1" -tags: - - name: Search - description: MetaCPAN Search Endpoints - # - name: Release - # description: Distribution Release Endpoints -schemes: - - "http" - - "https" -consumes: - - "application/json" -produces: - - "application/json" -paths: - /search/web: - $ref: "requests/search.yml#/search_web" - /search/first: - $ref: "requests/search.yml#/search_first" - /cover/{name}: - $ref: "requests/cover.yml#/cover" diff --git a/t/api/controller/admin.t b/t/api/controller/admin.t deleted file mode 100644 index d4921ac7b..000000000 --- a/t/api/controller/admin.t +++ /dev/null @@ -1,39 +0,0 @@ -use strict; -use warnings; -use lib 't/lib'; - -use Test::Mojo; -use Test::More; - -local $ENV{MOJO_SECRET} = 'Magritte'; -local $ENV{GITHUB_KEY} = 'foo'; -local $ENV{GITHUB_SECRET} = 'bar'; - -subtest 'authentication enabled' => sub { - my $t = Test::Mojo->new('MetaCPAN::API'); - $t->post_ok('/admin/enqueue'); - $t->header_is( Location => '/auth/github/authenticate' ); - $t->status_is(302); -}; - -subtest 'index release' => sub { - local $ENV{FORCE_ADMIN_AUTH} = 'tester'; - my $t = Test::Mojo->new('MetaCPAN::API'); - $t->get_ok('/admin/index-release'); - $t->status_is(200); -}; - -subtest 'search identities' => sub { - local $ENV{FORCE_ADMIN_AUTH} = 'tester'; - my $t = Test::Mojo->new('MetaCPAN::API'); - $t->get_ok('/admin/identity-search-form'); - $t->status_is(200); - - $t->post_ok( '/admin/search-identities' => form => - { name => 'pause', key => 'MO' } ); - $t->content_like(qr/\bMO\b/); - $t->content_like(qr/\bpause\b/); - $t->status_is(200); -}; - -done_testing(); diff --git a/t/api/controller/cover.t b/t/api/controller/cover.t deleted file mode 100644 index f8933777e..000000000 --- a/t/api/controller/cover.t +++ /dev/null @@ -1,56 +0,0 @@ -use Mojo::Base -strict; - -use lib 't/lib'; - -use MetaCPAN::TestServer (); -use Test::Mojo (); -use Test::More; - -my $server = MetaCPAN::TestServer->new; - -my $t = Test::Mojo->new( - 'MetaCPAN::API' => { - es => $server->es_client, - secret => 'just a test', - } -); - -my %expect = ( - 'MetaFile-Both-1.1' => { - criteria => { - branch => '12.50', - condition => '0.00', - statement => '63.64', - subroutine => '71.43', - total => '46.51', - }, - distribution => 'MetaFile-Both', - release => 'MetaFile-Both-1.1', - url => 'http://cpancover.com/latest/MetaFile-Both-1.1/index.html', - version => '1.1', - }, - 'Pod-With-Generator-1' => { - criteria => { - branch => '78.95', - condition => '46.67', - statement => '95.06', - subroutine => '100.00', - total => '86.58', - }, - distribution => 'Pod-With-Generator', - release => 'Pod-With-Generator-1', - url => 'http://cpancover.com/latest/Pod-With-Generator-1/index.html', - version => '1', - }, -); - -for my $release ( keys %expect ) { - my $expected = $expect{$release}; - subtest "Check $release" => sub { - - $t->get_ok("/v1/cover/$release")->status_is(200)->json_is($expected) - ->or( sub { diag $t->tx->res->dom } ); - - }; -} -done_testing; diff --git a/t/api/controller/search/first.t b/t/api/controller/search/first.t deleted file mode 100644 index 603182832..000000000 --- a/t/api/controller/search/first.t +++ /dev/null @@ -1,14 +0,0 @@ -use Mojo::Base -strict; - -use Test::Mojo (); -use Test::More; - -my $t = Test::Mojo->new('MetaCPAN::API'); - -$t->get_ok( '/v1/search/first', form => { q => 'Versions::PkgVar' } ) - ->status_is(200)->json_like( '/release' => qr/Versions-(?:\d+)/ ); - -$t->get_ok( '/v1/search/first', form => { q => 'DOESNOTEXISTS' } ) - ->status_is(404)->content_is(''); - -done_testing; diff --git a/t/api/controller/search/web.t b/t/api/controller/search/web.t deleted file mode 100644 index 377b7fae0..000000000 --- a/t/api/controller/search/web.t +++ /dev/null @@ -1,73 +0,0 @@ -use Mojo::Base -strict; - -use Mojo::JSON qw( false true ); -use Test::Mojo; -use Test::More; - -# Note: we need a release marked as status => latest -# so we're using Versions::PkgVar for now -# perhaps it should be smarter later and find one to try? - -my $t = Test::Mojo->new('MetaCPAN::API'); - -subtest 'collapsed' => sub { - $t->get_ok( '/v1/search/web', form => { q => 'Versions::PkgVar' } ) - ->status_is(200)->json_is( '/collapsed' => true ); - - $t->get_ok( '/v1/search/web', form => { q => 'module:Versions::PkgVar' } ) - ->status_is(200)->json_is( '/collapsed' => false ); - - $t->get_ok( '/v1/search/web', - form => { q => 'module:Versions::PkgVar', collapsed => 1 } ) - ->status_is(200)->json_is( '/collapsed' => true ); - - $t->get_ok( '/v1/search/web', form => { q => 'dist:Versions' } ) - ->status_is(200)->json_is( '/collapsed' => false ); - - $t->get_ok( '/v1/search/web', - form => { q => 'dist:Versions', collapsed => 1 } )->status_is(200) - ->json_is( '/collapsed' => true ); - - $t->get_ok( '/v1/search/web', form => { q => 'distribution:Versions' } ) - ->status_is(200)->json_is( '/collapsed' => false ); - - $t->get_ok( '/v1/search/web', - form => { q => 'distribution:Versions', collapsed => 1 } ) - ->status_is(200)->json_is( '/collapsed' => true ); -}; - -subtest 'paging' => sub { - my $q = 'this'; - $t->get_ok( '/v1/search/web', form => { q => $q } )->status_is(200); - my $json = $t->tx->res->json; - my $total = $json->{total}; - - if ( $total <= 1 ) { - cmp_ok @{ $json->{results} }, '==', $total, - 'results agree with total'; - diag "Only one search result, skipping remaining paging tests\n"; - return; - } - - diag "Testing paging with $total results\n"; - cmp_ok @{ $json->{results} }, '<=', $total, 'results agree with total'; - - # shrink the page size to one, test limit - $t->get_ok( '/v1/search/web', form => { q => $q, size => 1 } ) - ->status_is(200)->json_is( '/total' => $total, 'total is unchanged' ); - $json = $t->tx->res->json; - cmp_ok @{ $json->{results} }, '==', 1, 'results has been limited by size'; - my $first = $json->{results}[0]{hits}[0]{id}; - - # keep the page size as one, test offset - $t->get_ok( '/v1/search/web', form => { q => $q, size => 1, from => 1 } ) - ->status_is(200)->json_is( '/total' => $total, 'total is unchanged' ); - $json = $t->tx->res->json; - cmp_ok @{ $json->{results} }, '==', 1, 'results has been limited by size'; - my $next = $json->{results}[0]{hits}[0]{id}; - - isnt $first, $next, 'got a different result'; -}; - -done_testing; - diff --git a/templates/admin/identity_search_form.html.ep b/templates/admin/identity_search_form.html.ep deleted file mode 100644 index 43fbf5e77..000000000 --- a/templates/admin/identity_search_form.html.ep +++ /dev/null @@ -1,13 +0,0 @@ -
- - - - Identity value: - - -
diff --git a/templates/admin/index.html.ep b/templates/admin/index.html.ep deleted file mode 100644 index a39edf7f1..000000000 --- a/templates/admin/index.html.ep +++ /dev/null @@ -1,8 +0,0 @@ - diff --git a/templates/admin/search_identities.html.ep b/templates/admin/search_identities.html.ep deleted file mode 100644 index 3428abc44..000000000 --- a/templates/admin/search_identities.html.ep +++ /dev/null @@ -1,3 +0,0 @@ -display results below: -<%= stash('user_data')->{identity}[0]{name} %> -<%= stash('user_data')->{identity}[0]{key} %> diff --git a/templates/layouts/default.html.ep b/templates/layouts/default.html.ep deleted file mode 100644 index fbf9c181c..000000000 --- a/templates/layouts/default.html.ep +++ /dev/null @@ -1,5 +0,0 @@ -

-
MetaCPAN Admin
-

- -<%= content %> diff --git a/templates/queue/index_release.html.ep b/templates/queue/index_release.html.ep deleted file mode 100644 index e69de29bb..000000000 From 6777b50e7cfb33a78775d54c2e7b145c7fda57e5 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:22:36 +0200 Subject: [PATCH 11/15] stop testing Minion setup Our tests for minion don't really check much, and add a dependency on Postgres. Eventually these tests can return in ingest once it is ready. --- t/api/queue.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/api/queue.t b/t/api/queue.t index b70e051e9..492244b38 100644 --- a/t/api/queue.t +++ b/t/api/queue.t @@ -2,10 +2,10 @@ use strict; use warnings; use lib 't/lib'; +use Test::More skip_all => 'disabling Minion tests to avoid needing postgres'; use MetaCPAN::DarkPAN (); use Path::Tiny qw( path ); use Test::Mojo; -use Test::More; my $t = Test::Mojo->new('MetaCPAN::API'); my $app = $t->app; From fe3e94f711201cd6388276ce7a4ed09740f6b7f0 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:24:36 +0200 Subject: [PATCH 12/15] add ${VAR} replacement support to config This supports ${VAR}, ${VAR:-default}, and ${VAR:+set} variable replacements to allow overridable config variables. --- lib/MetaCPAN/Server/Config.pm | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Config.pm b/lib/MetaCPAN/Server/Config.pm index 4bf51f97a..a8b032c7d 100644 --- a/lib/MetaCPAN/Server/Config.pm +++ b/lib/MetaCPAN/Server/Config.pm @@ -36,7 +36,28 @@ sub _zomg { my $v = Data::Visitor::Callback->new( plain_value => sub { return unless defined $_; - s{__HOME__}{$root}ge; + s{ + (__HOME__) + | + (\$\{([^\}]+)\}) + }{ + defined $1 ? $root + : defined $2 ? do { + my $var = $3; + if ($var =~ s{:-(.*)}{}) { + my $sub = $1; + $ENV{$var} // $1; + } + elsif ($var =~ s{:\+(.*)}{}) { + my $sub = $1; + $ENV{$var} ? $sub : ''; + } + else { + $ENV{$var} // ''; + } + } + : '' + }gex; } ); $v->visit($c); From 902192df1bcbce3a5640f6990ef148c569e01385 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:27:00 +0200 Subject: [PATCH 13/15] add testing profile to docker compose --- docker-compose.yml | 47 ++++++++++++++++++++++++++++++++++++ metacpan_server_testing.yaml | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 965fbaa92..99066703d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,3 +15,50 @@ services: watch: - path: ./cpanfile action: rebuild + + api-test: + profiles: + - test + depends_on: + elasticsearch-test: + condition: service_healthy + build: + context: . + target: test + environment: + NET_ASYNC_HTTP_MAXCONNS: 1 + COLUMNS: 80 + ES: http://elasticsearch-test:9200 + HARNESS_ACTIVE: 1 + # Instantiate Catalyst models using metacpan_server_testing.conf + METACPAN_SERVER_CONFIG_LOCAL_SUFFIX: testing + MINICPAN: /CPAN + DEVEL_COVER_OPTIONS: +ignore,^t/|^test-data/|^etc/|^local/ + networks: + - elasticsearch + volumes: + - type: volume + source: elasticsearch-test + target: /usr/share/elasticsearch/data + + elasticsearch-test: + profiles: + - test + platform: linux/amd64 + image: elasticsearch:2.4 + environment: + - discovery.type=single-node + healthcheck: + timeout: 5s + start_period: 60s + test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] + ports: + - "9200" + networks: + - elasticsearch + +networks: + elasticsearch: + +volumes: + elasticsearch-test: diff --git a/metacpan_server_testing.yaml b/metacpan_server_testing.yaml index f47e101d5..23b6e4406 100644 --- a/metacpan_server_testing.yaml +++ b/metacpan_server_testing.yaml @@ -8,7 +8,7 @@ source_base: var/t/tmp/source elasticsearch_servers: client: '2_0::Direct' - nodes: http://elasticsearch_test:9200 + nodes: ${ES:-http://elasticsearch_test:9200} minion_dsn: "postgresql://metacpan:t00lchain@pghost:5432/minion_queue" From 62652423cb5507eb80189520ed8d908f4b68fa61 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 09:02:02 +0200 Subject: [PATCH 14/15] update docker build and push to use our standard setup Use a single workflow for docker building and pushing, and use the reusable action to make it standardized. Leaving in a commented out step to update in k8s. Once it is set up, we can uncomment it. --- .github/workflows/build-container.yml | 46 +++++++++++++++++++ .../workflows/build-deployment-container.yml | 26 ----------- .../workflows/build-production-container.yml | 24 ---------- 3 files changed, 46 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/build-container.yml delete mode 100644 .github/workflows/build-deployment-container.yml delete mode 100644 .github/workflows/build-production-container.yml diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 000000000..817561902 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,46 @@ +name: Build container +on: + push: + branches: + - master + - staging + - prod + pull_request: + types: [opened, synchronize, labeled] + branches: + - master + workflow_dispatch: +jobs: + docker-build: + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-container') + runs-on: ubuntu-22.04 + name: Docker Build and Push + steps: + - name: Generate Auth Token + uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: metacpan + - uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - uses: metacpan/metacpan-actions/docker-build-push@master + id: build-push + with: + docker_hub_username: ${{ secrets.DOCKER_HUB_USER }} + docker_hub_password: ${{ secrets.DOCKER_HUB_TOKEN }} + ghcr_username: ${{ github.repository_owner }} + ghcr_password: ${{ secrets.GITHUB_TOKEN }} + test-target: test +# - name: Update deployed image +# if: steps.find-tag-names.outputs.latest +# uses: metacpan/metacpan-actions/update-deployed-tag:master +# with: +# token: ${{ steps.app-token.outputs.token }} +# app: api +# environment: prod +# base-tag: ${{ steps.find-tag-names.outputs.latest }} +# tag: ${{ steps.find-tag-names.outputs.sha }} + diff --git a/.github/workflows/build-deployment-container.yml b/.github/workflows/build-deployment-container.yml deleted file mode 100644 index 721d46dc7..000000000 --- a/.github/workflows/build-deployment-container.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: Build deployment container -on: - push: - branches: - - prod - - staging - workflow_dispatch: -jobs: - docker: - runs-on: ubuntu-22.04 - name: Docker push SHA - steps: - - uses: actions/checkout@v4 - - name: docker build - run: docker build . -t metacpan/metacpan-api:$GITHUB_SHA - - name: run Perl tests - run: docker run -i metacpan/metacpan-api carton exec prove -lr --jobs 2 t - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: push build to Docker hub - run: docker push metacpan/metacpan-api:$GITHUB_SHA - diff --git a/.github/workflows/build-production-container.yml b/.github/workflows/build-production-container.yml deleted file mode 100644 index 584f9b54f..000000000 --- a/.github/workflows/build-production-container.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: Build Production Container -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - docker: - runs-on: ubuntu-22.04 - name: Docker push latest - steps: - - uses: actions/checkout@v4 - - name: docker build - run: docker build . -t metacpan/metacpan-api:latest - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: push build to Docker hub - run: docker push metacpan/metacpan-api:latest - if: success() && github.ref == 'refs/heads/master' From 74b9d098920917f2d8e9d1f62ea37b66157412d3 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:27:12 +0200 Subject: [PATCH 15/15] update circleci config for new docker compose setup This repo now has its own docker compose config, so it doesn't need to pull in a separate repo for testing. This simplifies the CI setup, and makes future updates easier as they don't have to coordinate across multiple repos. --- .circleci/config.yml | 62 ++++++++------------------------------------ 1 file changed, 11 insertions(+), 51 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc0c7ae68..0aa915976 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,64 +15,24 @@ jobs: resource_class: large steps: - run: - name: docker-compose version - command: docker-compose --version + name: docker compose version + command: docker compose version + - checkout - run: + name: create coverage directory command: | - git clone https://github.com/metacpan/metacpan-docker.git - cd metacpan-docker - name: metacpan-docker checkout - - checkout: - path: metacpan-docker/src/metacpan-api + mkdir cover_db + chmod o+w cover_db - run: + name: docker compose build command: | - pushd metacpan-docker - ./bin/metacpan-docker init - name: clone missing repositories + docker compose --profile test build api-test - run: - command: | - pushd metacpan-docker - docker-compose build --build-arg CPM_ARGS='--with-test' api_test - name: compose build - - run: - command: | - pushd metacpan-docker - ./bin/metacpan-docker init - docker-compose --verbose up -d api_test - name: compose up - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test cpm install -g Devel::Cover - name: install Devel::Cover - # Since we're running docker-compose -d, we don't actually know if - # Elasticsearch is available at the time this build step begins. We - # probably need to wait for it here, so we'll add our own check. - - run: - command: | - pushd metacpan-docker - ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test - name: wait for ES - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test env HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lr --jobs 4 t name: run tests with coverage + command: | + docker compose --profile test run --env HARNESS_PERL_SWITCHES=-MDevel::Cover -v ./cover_db:/app/cover_db/ api-test bash -c 'prove -lr -j4 t && cover -report json' # We are relying on environment variables from the host to be available when # we publish the report, so we publish from the host rather than trying # to propagate env variables to the container. - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test cover -report json - name: create coverage report - codecov/upload: - file: metacpan-docker/src/metacpan-api/cover_db/cover.json - - run: - command: | - pushd metacpan-docker - docker-compose logs - docker stats --no-stream - docker ps -a | head - name: docker-compose logs - when: on_fail + file: cover_db/cover.json