Skip to content

Commit

Permalink
Extra filters to /app/package (#535)
Browse files Browse the repository at this point in the history 
* Add test for official package
* Add "ignore" filters to /app/package
* db: Tweak popular_os.mask for 15.6 and Slowroll
  • Loading branch information
@andrii-suse
andrii-suse authored Nov 22, 2024
1 parent 1ebff13 commit 6504881
Show file tree
Hide file tree
Showing 8 changed files with 212 additions and 48 deletions.
28 changes: 26 additions & 2 deletions assets/javascripts/package.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ var pkg_param_official;
var pkg_param_os;
var pkg_param_os_ver;
var pkg_param_repo;
var pkg_param_ign_path;
var pkg_param_ign_file;

function initPackageParams() {

Expand All @@ -18,6 +20,8 @@ function initPackageParams() {
pkg_param_os = urlParams.get('os');
pkg_param_os_ver = urlParams.get('os_ver');
pkg_param_repo = urlParams.get('repo');
pkg_param_ign_path = urlParams.get('ignore_path');
pkg_param_ign_file = urlParams.get('ignore_file');

if (pkg_param_pkg) {
( document.getElementById("packag") || {} ).value = pkg_param_pkg;
Expand All @@ -37,6 +41,12 @@ function initPackageParams() {
if (pkg_param_repo) {
document.getElementById("repo").value = pkg_param_repo;
}
if (pkg_param_ign_path) {
document.getElementById("ign_path").value = pkg_param_ign_path;
}
if (pkg_param_ign_file) {
document.getElementById("ign_file").value = pkg_param_ign_file;
}

}

Expand All @@ -54,6 +64,8 @@ function setupPackages() {
pkg_param_os = document.getElementById("os").value;
pkg_param_os_ver = document.getElementById("os_ver").value;
pkg_param_repo = document.getElementById("repo").value;
pkg_param_ign_path = document.getElementById("ign_path").value;
pkg_param_ign_file = document.getElementById("ign_file").value;

var dataTable = table.DataTable({
ajax: {
Expand All @@ -64,7 +76,9 @@ function setupPackages() {
"official": pkg_param_official,
"os": pkg_param_os,
"os_ver": pkg_param_os_ver,
"repo": pkg_param_repo
"repo": pkg_param_repo,
"ignore_path": pkg_param_ign_path,
"ignore_file": pkg_param_ign_file,
},
},
deferRender: true,
Expand All @@ -91,6 +105,12 @@ function setupPackages() {
if (pkg_param_repo) {
get.push(['repo', htmlEscape(pkg_param_repo)]);
}
if (pkg_param_ign_path) {
get.push(['ignore_path', htmlEscape(pkg_param_ign_path)]);
}
if (pkg_param_ign_file) {
get.push(['ignore_file', htmlEscape(pkg_param_ign_file)]);
}
var getstr = '';
for (var i = 0; i < get.length; i++) {
if (getstr) {
Expand Down Expand Up @@ -129,6 +149,8 @@ function setupPackageLocations(name) {
pkg_param_os = document.getElementById("os").value;
pkg_param_os_ver = document.getElementById("os_ver").value;
pkg_param_repo = document.getElementById("repo").value;
pkg_param_ign_path = document.getElementById("ign_path").value;
pkg_param_ign_file = document.getElementById("ign_file").value;

var dataTable = table.DataTable({
ajax: {
Expand All @@ -139,7 +161,9 @@ function setupPackageLocations(name) {
"official": pkg_param_official,
"os": pkg_param_os,
"os_ver": pkg_param_os_ver,
"repo": pkg_param_repo
"repo": pkg_param_repo,
"ignore_path": pkg_param_ign_path,
"ignore_file": pkg_param_ign_file,
},
},
deferRender: true,
Expand Down
26 changes: 23 additions & 3 deletions lib/MirrorCache/WebAPI/Controller/Rest/Metapkg.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ sub search_locations {
my $p_os = $self->param('os');
my $p_os_ver = $self->param('os_ver');
my $p_repo = $self->param('repo');
my $p_ign_path = $self->param('ignore_path');
my $p_ign_file = $self->param('ignore_file');

my $sql_from = <<'END_SQL';
select metapkg.name, folder.path as path, file.name as file, file.size as size, file.mtime as time
Expand All @@ -55,7 +57,7 @@ END_SQL


if ($p_official) {
$sql_from = $sql_from . "\njoin project on folder.path like concat(project.name, '%') and project.prio > 10\n";
$sql_from = $sql_from . "\njoin project on folder.path like concat(project.path, '%') and project.prio > 10\n";
}
if ($p_os) {
$sql_from = $sql_from . "\njoin popular_os os on folder.path ~ os.mask and (coalesce(os.neg_mask,'') = '' or not folder.path ~ os.neg_mask) and os.name = ?\n";
Expand All @@ -75,6 +77,14 @@ END_SQL
push @parms, $p_repo;
push @parms, ($arch ? $arch : '%');
}
if ($p_ign_path) {
$sql_where = "$sql_where and folder.path not like concat('%', ?::text, '%')";
push @parms, $p_ign_path;
}
if ($p_ign_file) {
$sql_where = "$sql_where and file.name not like concat('%', ?::text, '%')";
push @parms, $p_ign_file;
}

my $sql = $sql_from . "\n" . $sql_where;

Expand All @@ -95,6 +105,8 @@ sub search {
my $p_os = $self->param('os');
my $p_os_ver = $self->param('os_ver');
my $p_repo = $self->param('repo');
my $p_ign_path = $self->param('ignore_path');
my $p_ign_file = $self->param('ignore_file');

my $sql_from = <<'END_SQL';
select distinct metapkg.name
Expand All @@ -110,14 +122,14 @@ END_SQL
$arch = "%$p";
}

if ($p_official || $p_os || $p_repo) {
if ($p_official || $p_os || $p_repo || $p_ign_path) {
$sql_from = "$sql_from\njoin folder on pkg.folder_id = folder.id";
}

# and file.name like concat(metapkg.name, '-%')

if ($p_official) {
$sql_from = $sql_from . "\njoin project on folder.path like concat(project.name, '%') and project.prio > 10\n";
$sql_from = $sql_from . "\njoin project on folder.path like concat(project.path, '%') and project.prio > 10\n";
}
if ($p_os) {
$sql_from = $sql_from . "\njoin popular_os os on folder.path ~ os.mask and (coalesce(os.neg_mask,'') = '' or not folder.path ~ os.neg_mask) and os.name = ?\n";
Expand All @@ -133,6 +145,14 @@ END_SQL
push @parms, $p_repo;
push @parms, ($arch ? $arch : '%');
}
if ($p_ign_path) {
$sql_where = "$sql_where and folder.path not like concat('%', ?::text, '%')";
push @parms, $p_ign_path;
}
if ($p_ign_file) {
$sql_where = "$sql_where and metapkg.name not like concat('%', ?::text, '%')";
push @parms, $p_ign_file;
}

my $sql = $sql_from . "\n" . $sql_where;

Expand Down
3 changes: 3 additions & 0 deletions lib/MirrorCache/resources/migrations/Pg.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,4 +441,7 @@ create table if not exists pkg (
);

create index if not exists pkg_metapkg_id_idx on pkg(metapkg_id);
-- 40 up
update popular_os set mask = '.*[lL]eap(/|_)(([1-9][0-9])(.|_)([0-9])?(-test|-Current)?)/.*|(.*/(16|15|12|43|42).(0|1|2|3|4|5|6)/.*)' where id = 4;
insert into popular_os(id,name,mask) select 10, 'slowroll', '.*/[Ss]lowroll/.*' on conflict do nothing;

4 changes: 3 additions & 1 deletion lib/MirrorCache/resources/migrations/mysql.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,4 +454,6 @@ create table if not exists pkg (
);

create index if not exists pkg_metapkg_id_idx on pkg(metapkg_id);

-- 40 up
update popular_os set mask = '.*[lL]eap(/|_)(([1-9][0-9])(.|_)([0-9])?(-test|-Current)?)/.*|(.*/(16|15|12|43|42).(0|1|2|3|4|5|6)/.*)' where id = 4;
insert into popular_os(id,name,mask) select 10, 'slowroll', '.*/[Ss]lowroll/.*' on duplicate key update id=id;
50 changes: 50 additions & 0 deletions t/environ/24-pkg-project.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#!lib/test-in-container-environ.sh
set -ex

mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0

$mc/gen_env MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL

$mc/start
$mc/status

ap8=$(environ ap8)
ap7=$(environ ap7)

files=(
/folder1/x86_64/mypkg-1.1-1.1.x86_64.rpm
/folder2/x86_64/yourpkg-1.1-1.1.x86_64.rpm
)


for f in ${files[@]}; do
for x in $mc $ap7 $ap8; do
mkdir -p $x/dt${f%/*}
echo 1111111111 > $x/dt$f
done
done

$ap7/start
$ap8/start

$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'"

$mc/sql "insert into project(name,path,prio) select 'proj','/folder2', 100"

for f in ${files[@]}; do
$mc/curl -Is /download$f
done

$mc/backstage/job -e folder_sync -a '["/folder1/x86_64"]'
$mc/backstage/job -e folder_sync -a '["/folder2/x86_64"]'
$mc/backstage/shoot

$mc/curl /rest/search/packages?official=1 | grep yourpkg

rc=0
$mc/curl /rest/search/packages?official=1 | grep mypkg || rc=1
test $rc -gt 0

echo success
9 changes: 9 additions & 0 deletions t/environ/24-pkg.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,13 @@ done
$mc/sql_test 6 == "select count(*) from pkg"
$mc/sql_test 6 == "select count(*) from metapkg"

$mc/curl /rest/search/packages

$mc/curl /rest/search/packages?ignore_path=tumbleweed
$mc/curl /rest/search/packages?ignore_file=python
$mc/curl "/rest/search/packages?ignore_file=python&ignore_path=tumbleweed"

$mc/curl "/rest/search/package_locations?package=xmltooling-schemas"
$mc/curl "/rest/search/package_locations?package=xmltooling-schemas&ignore_path=shibboleth"

echo success
67 changes: 45 additions & 22 deletions templates/app/package/index.html.ep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,20 @@ input[type = text] {
<div class="row">
<div class="col-sm-12">
<h2><%= title %></h2>

<div class="container-fluid" id="content">
<div class="card">
<div class="card-header">Specify search parameters</div>
<div class="card-body">
<form action="javascript:setupPackages()">
<div>
<div class = row>
<div class = "row">
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="packag">Package</label>
</div>
<div class = col75>
<input id="packag" name="packag" type="text">
</div>
</div>
<div class = row>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="arch">Arch</label>
</div>
Expand All @@ -67,7 +65,25 @@ input[type = text] {
</datalist>
</div>
</div>
<div class = row>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="repo">Repository</label>
</div>
<div class = col75>
<input id="repo" name="repo" type="text" list="repolist">
<datalist id="repolist">
<option value="15.6">15.6</option>
<option value="openSUSE_Tumbleweed">openSUSE_Tumbleweed</option>
<option value="openSUSE_Leap_15.6">openSUSE_Leap_15.6</option>
<option value="15.6">15.5</option>
<option value="openSUSE_Leap_15.5">openSUSE_Leap_15.5</option>
<option value="16.0">16.0</option>
</datalist>
</div>
</div>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="os">OS</label>
</div>
Expand All @@ -86,7 +102,8 @@ input[type = text] {
</datalist>
</div>
</div>
<div class = row>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="os_ver">OS Version</label>
</div>
Expand All @@ -99,36 +116,42 @@ input[type = text] {
</datalist>
</div>
</div>
<div class = row>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="repo">Repository</label>
<label class="form-label" for="official">Official</label>
</div>
<div class = col75>
<input id="repo" name="repo" type="text" list="repolist">
<datalist id="repolist">
<option value="15.6">15.6</option>
<option value="openSUSE_Tumbleweed">openSUSE_Tumbleweed</option>
<option value="openSUSE_Leap_15.6">openSUSE_Leap_15.6</option>
<option value="15.6">15.5</option>
<option value="openSUSE_Leap_15.5">openSUSE_Leap_15.5</option>
<option value="16.0">16.0</option>
<input id="official" name="official" type="checkbox">
</div>
</div>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="ign_path">Ignore path</label>
</div>
<div class = col75>
<input id="ign_path" name="ign_path" type="text" list="ign_pathlist">
<datalist id="ign_pathlist">
<option value="/repositories">/repositories</option>
<option value="/repositories/home:">/repositories/home:</option>
</datalist>
</div>
</div>
<div class = row>
<br>
<div class = "col-xl-4">
<div class = col25>
<label class="form-label" for="official">Official</label>
<label class="form-label" for="ign_file">Ignore files</label>
</div>
<div class = col75>
<input id="official" name="official" type="checkbox">
<input id="ign_file" name="ign_file" type="text">
</div>
</div>
</div>
<div class="mb-3">
<input class="btn btn-primary" type="submit" value="Search"/>
</div>
</form>
</div>
</div>
</div>
<table id="packages" class="admintable table table-striped">
Expand Down
Loading

0 comments on commit 6504881

Please sign in to comment.