#!/usr/bin/perl -w
#
# Copyright (c) 2006, 2007 Michael Schroeder, Novell Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################
#
# The Publisher. Create repositories and push them to our mirrors.
#

BEGIN {
  my ($wd) = $0 =~ m-(.*)/- ;
  $wd ||= '.';
  unshift @INC,  "$wd/build";
  unshift @INC,  "$wd";
}

use Digest;
use Digest::MD5 ();
use Digest::SHA ();
use XML::Structured ':bytes';
use XML::Simple ();
use POSIX;
use Fcntl qw(:DEFAULT :flock);
use Data::Dumper;
use Storable ();
use MIME::Base64;
use JSON::XS ();

use BSConfiguration;
use BSCando;
use BSRPC ':https';
use BSUtil;
use BSDBIndex;
use Build;
use BSDB;
use BSXML;
use BSNotify;
use BSVerify;
use BSStdRunner;
use BSUrlmapper;
use BSPGP;
use BSHTTP;
use BSRekor;
use BSRepServer::Containerinfo;
use BSPublisher::Container;
use BSPublisher::Helm;
use BSPublisher::Util;
use BSPublisher::Blobstore;

use strict;

my $maxchild;
my $maxchild_flavor;
$maxchild = $BSConfig::publish_maxchild if defined $BSConfig::publish_maxchild;
$maxchild_flavor = $BSConfig::publish_maxchild_flavor if defined $BSConfig::publish_maxchild_flavor;

my $reporoot = "$BSConfig::bsdir/build";
my $eventdir = "$BSConfig::bsdir/events";
my $extrepodir = "$BSConfig::bsdir/repos";
my $extrepodir_sync = "$BSConfig::bsdir/repos_sync";
my $uploaddir = "$BSConfig::bsdir/upload";
my $rundir = $BSConfig::rundir || $BSConfig::rundir || "$BSConfig::bsdir/run";

my $extrepodb = "$BSConfig::bsdir/db/published";

my $myeventdir = "$eventdir/publish";

my @binsufs = qw{rpm udeb ddeb deb pkg.tar.gz pkg.tar.xz pkg.tar.zst};
my $binsufsre = join('|', map {"\Q$_\E"} @binsufs);
my @binsufsrsync = map {"--include=*.$_"} @binsufs;
my $binarchs = join("|", keys(%BSCando::knownarch));

my $trackercacheversion = 2;

my $testmode;

# convert hooks with filenames to real code refs
if ($BSConfig::createrepo_rpmmd_hook && !ref($BSConfig::createrepo_rpmmd_hook)) {
  $BSConfig::createrepo_rpmmd_hook = require($BSConfig::createrepo_rpmmd_hook);
  die("createrepo_rpmmd_hook did not return a code ref\n") unless ref($BSConfig::createrepo_rpmmd_hook);
}
if ($BSConfig::publisher_compile_content_hook && !ref($BSConfig::publisher_compile_content_hook)) {
  $BSConfig::publisher_compile_content_hook = require($BSConfig::publisher_compile_content_hook);
  die("publisher_compile_content_hook did not return a code ref\n") unless ref($BSConfig::publisher_compile_content_hook);
}

=head1 qsystem - secure execution of system calls with output redirection

 Examples:

   qsystem('stdout', $tempfile, $decomp, $in);

   qsystem('chdir', $extrep, 'stdout', 'Packages.new', 'dpkg-scanpackages', '-m', '.', '/dev/null')

=cut

sub qsystem {
  my @args = @_;
  my $pid;
  local (*RH, *WH);
  if ($args[0] eq 'echo') {
    pipe(RH, WH) || die("pipe: $!\n");
  }
  if (!($pid = xfork())) {
    if ($args[0] eq 'echo') {
      close WH;
      open(STDIN, "<&RH");
      close RH;
      splice(@args, 0, 2);
    }
    if ($args[0] eq 'chdir') {
      chdir($args[1]) || die("chdir $args[1]: $!\n");
      splice(@args, 0, 2);
    }
    if ($args[0] eq 'stdout') {
      if ($args[1] ne '') {
        open(STDOUT, '>', $args[1]) || die("$args[1]: $!\n");
      }
      splice(@args, 0, 2);
    } else {
      open(STDOUT, ">/dev/null");
    }
    eval {
      exec(@args);
      die("$args[0]: $!\n");
    };
    warn($@) if $@;
    exit 1;
  }
  if ($args[0] eq 'echo') {
    close RH;
    print WH $args[1];
    close WH;
  }
  waitpid($pid, 0) == $pid || die("waitpid $pid: $!\n");
  return $?;
}

sub fillpkgdescription {
  my ($pkg, $extrep, $repoinfo, $name) = @_;
  my $binaryorigins = $repoinfo->{'binaryorigins'} || {};
  my $hit;
  for my $p (sort keys %$binaryorigins) {
    next if $p =~ /src\.rpm$/;
    next unless $p =~ /\/\Q$name\E/;
    my ($pa, $pn) = split('/', $p, 2);
    if ($pn =~ /^\Q$name\E-([^-]+-[^-]+)\.[^\.]+\.rpm$/) {
      $hit = $p;
      last;
    }
    if ($pn =~ /^\Q$name\E_([^_]+)_[^_]+\.[ud]?deb$/) {
      $hit = $p;
      last;
    }
  }
  return unless $hit;
  eval {
    my $data = Build::query("$extrep/$hit", 'description' => 1);
    $pkg->{'description'} = str2utf8($data->{'description'});
    $pkg->{'summary'} = str2utf8($data->{'summary'}) if defined $data->{'summary'};
  };
}


############################################################################################

my @db_sync;
my $db_oldsync_read;
my $db_sync_append;

sub db_pickup {
  if (-s "$extrepodb.sync") {
    my $oldsync = BSUtil::retrieve("$extrepodb.sync");
    unshift @db_sync, @{$oldsync || []};
  }
}

sub db_open {
  my ($name) = @_;

  return undef unless $extrepodb;
  if (!$db_oldsync_read) {
    db_pickup();
    $db_oldsync_read = 1;
  }
  return {'name' => $name, 'index' => "$name/"};
}

sub db_updateindex_rel {
  my ($db, $rem, $add) = @_;
  push @db_sync, $db->{'name'}, $rem, $add;
}

sub db_store {
  my ($db, $k, $v) = @_;
  push @db_sync, $db->{'name'}, $k, $v;
}

sub db_sync {
  return undef unless $extrepodb;
  db_open('') unless $db_oldsync_read;
  return unless @db_sync;
  my $data = Storable::nfreeze(\@db_sync);
  my $ops = @db_sync;
  for (@db_sync) {
    $ops += @$_ if $_ && ref($_) eq 'ARRAY';
  }
  my $timeout = $ops / 30;
  $timeout = 60 if $timeout < 60;
  my $param = {
    'uri' => "$BSConfig::srcserver/search/published",
    'request' => 'POST',
    'maxredirects' => 3,
    'timeout' => $timeout,
    'headers' => [ 'Content-Type: application/octet-stream' ],
    'data' => $data,
  };
  print "    syncing database ($ops ops)\n";
  eval {
    BSRPC::rpc($param, undef, 'cmd=updatedb');
  };
  if ($@) {
    warn($@);
    mkdir_p($1) if $extrepodb =~ /^(.*)\//;
    if ($db_sync_append) {
      local *F;
      BSUtil::lockopen(\*F, '>>', "$extrepodb.sync");
      db_pickup();
      BSUtil::store("$extrepodb.sync.new$$", "$extrepodb.sync", \@db_sync);
      close F;
      @db_sync = ();
    } else {
      BSUtil::store("$extrepodb.sync.new", "$extrepodb.sync", \@db_sync);
    }
  } else {
    @db_sync = ();
    unlink("$extrepodb.sync") unless $db_sync_append;
  }
}

############################################################################################

sub updatebinaryindex {
  my ($db, $keyrem, $keyadd) = @_;

  my $index = $db->{'index'};
  $index =~ s/\/$//;
  my @add;
  for my $key (@{$keyadd || []}) {
    my $n;
    if ($key =~ /(?:^|\/)([^\/]+)-[^-]+-[^-]+\.[a-zA-Z][^\/\.\-]*\.rpm$/) {
      $n = $1;
    } elsif ($key =~ /(?:^|\/)([^\/]+)_([^\/]*)_[^\/]*\.[ud]?deb$/) {
      $n = $1;
    } elsif ($key =~ /(?:^|\/)([^\/]+)-[^-]+-[^-]+-[a-zA-Z][^\/\.\-]*\.pkg\.tar\.(?:gz|xz|zst)$/) {
      $n = $1;
    } else {
      next;
    }
    push @add, ["$index/name", $n, $key];
  }
  my @rem;
  for my $key (@{$keyrem || []}) {
    my $n;
    if ($key =~ /(?:^|\/)([^\/]+)-[^-]+-[^-]+\.[a-zA-Z][^\/\.\-]*\.rpm$/) {
      $n = $1;
    } elsif ($key =~ /(?:^|\/)([^\/]+)_([^\/]*)_[^\/]*\.[ud]?deb$/) {
      $n = $1;
    } elsif ($key =~ /(?:^|\/)([^\/]+)-[^-]+-[^-]+-[a-zA-Z][^\/\.\-]*\.pkg\.tar\.(?:gz|xz|zst)$/) {
      $n = $1;
    } else {
      next;
    }
    push @rem, ["$index/name", $n, $key];
  }
  db_updateindex_rel($db, \@rem, \@add);
}


##########################################################################

sub getpatterns {
  my ($projid) = @_;

  my $dir;
  eval {
    $dir = BSRPC::rpc("$BSConfig::srcserver/source/$projid/_pattern", $BSXML::dir);
  };
  if ($@) {
    warn($@);
    return [];
  }
  my @ret;
  my @args;
  push @args, "rev=$dir->{'srcmd5'}" if $dir->{'srcmd5'} && $dir->{'srcmd5'} ne 'pattern';
  for my $entry (@{$dir->{'entry'} || []}) {
    my $pat;
    eval {
      $pat = BSRPC::rpc("$BSConfig::srcserver/source/$projid/_pattern/$entry->{'name'}", undef, @args);
      # only patterns we can parse, please
      BSUtil::fromxml($pat, $BSXML::pattern);
    };
    if ($@) {
      warn("   pattern $entry->{'name'}: $@");
      next;
    }
    push @ret, {'name' => $entry->{'name'}, 'md5' => $entry->{'md5'}, 'data' => $pat};
  }
  print "    fetched ".@ret." patterns\n";
  return \@ret;
}

##########################################################################

sub getsigndata {
  my ($projid) = @_;

  my @args = ("withpubkey=1", "withalgo=1");
  push @args, "autoextend=1" if $BSConfig::sign;
  my $signkey = BSRPC::rpc("$BSConfig::srcserver/getsignkey", undef, "project=$projid", @args);
  my $pubkey;
  my $algo;
  if ($signkey) {
    ($signkey, $pubkey) = split("\n", $signkey, 2);
    undef $pubkey unless $pubkey && length($pubkey) > 10;	# not a valid pubkey
    $algo = $1 if $signkey && $signkey =~ s/^(\S+)://;
    undef $algo if $algo && $algo eq '?';
  }
  if (!$pubkey) {
    if ($BSConfig::sign_project && $BSConfig::sign) {
      local *S;
      open(S, '-|', $BSConfig::sign, '--project', $projid, '-p') || die("$BSConfig::sign: $!\n");;
      $pubkey = '';
      1 while sysread(S, $pubkey, 4096, length($pubkey));
      if (!close(S)) {
	print "sign -p failed: $?\n";
	$pubkey = undef;
      }
    } elsif ($BSConfig::keyfile) {
      if (-e $BSConfig::keyfile) {
        $pubkey = readstr($BSConfig::keyfile);
      } else {
        print "WARNING: configured sign key $BSConfig::keyfile does not exist\n";
      }
    }
  }
  if ($pubkey && !$algo) {
    # try to get algo from public key
    eval { $algo = BSPGP::pk2algo(BSPGP::unarmor($pubkey)) };
  }
  # setup sign arguments
  my $signargs = [];
  if ($signkey) {
    mkdir_p($uploaddir);
    writestr("$uploaddir/publisher.$$", undef, $signkey);
    push @$signargs, '-P', "$uploaddir/publisher.$$";
  }
  push @$signargs, '-h', 'sha256' if $algo && $algo eq 'rsa';
  return ($pubkey, $signargs);
}

##########################################################################

sub pickup_detached_signature {
  my ($data, $file) = @_;
  return unless $data->{'sigs'} && $data->{'pubkey'};
  my $d = readstr($file);
  my $sig = readstr("$file.asc");
  push @{$data->{'sigs'}}, { 'data' => $d, 'signature' => $sig, 'pubkey' => $data->{'pubkey'}, 'format' => 'pgp' };
}

##########################################################################

sub addsizechecksum {
  my ($filename, $d, $sum) = @_;

  local *F;
  open(F, '<', $filename) || return;
  $d->{'size'} = -s F;
  my %known = (
    'sha' => 'SHA-1',
    'sha1' => 'SHA-1',
    'sha256' => 'SHA-256',
    'sha512' => 'SHA-512',
  );
  if ($known{$sum}) {
    my $ctx = Digest->new($known{$sum});
    $ctx->addfile(\*F);
    $d->{'checksum'} = {'type' => $sum, '_content' => $ctx->hexdigest()};
  }
  close F;
}

sub create_appdata_files {
  my ($dir, $appdatas) = @_;

  $appdatas = BSUtil::clone($appdatas);
  print "    creating appdata files\n";
  my %ids;
  my %written;
  mkdir_p("$dir/app-icons");
  for my $app (@{$appdatas->{'application'} || []}, @{$appdatas->{'component'} || []}) {
    for my $icon (@{$app->{'icon'} || []}) {
      my $iconname = ($icon->{'name'} || [])->[0];
      my $filecontent = ($icon->{'filecontent'} || [])->[0];
      next unless $iconname && $icon->{'filecontent'};
      next if $iconname =~ /\//s;
      my %files;
      for my $filecontent (@{$icon->{'filecontent'}}) {
	if (ref($filecontent)) {
	  next unless $filecontent->{'content'};
	  $files{$filecontent->{'file'} || $iconname} ||= $filecontent->{'content'};
	} else {
	  $files{$iconname} ||= $filecontent if $filecontent;
	}
      }
      next unless %files;
      my $fn;
      for my $size (qw(32 48 64 24)) {
	my @c = grep {/${size}x$size/} sort(keys %files);
	my @ch = grep {/\/hicolor\//} @c;
	@c = @ch if @ch;
	$fn = $c[0];
	last if $fn;
      }
      $fn ||= (sort(keys %files))[0];
      if ($iconname !~ /\./) {
	next unless $fn =~ /(\.[^\.\/]+)$/;
	$iconname .= $1;
      }
      if (!$written{$iconname}) {
	writestr("$dir/app-icons/$iconname", undef, decode_base64($files{$fn}));
	$written{$iconname} = 1;
      }
      $icon = { 'type' => 'cached', 'content' => $iconname};
    }
  }
  unlink("$dir/app-icons.tar");
  if (%written) {
    qsystem('chdir', "$dir/app-icons", 'tar', 'cf', '../app-icons.tar', '.') && die("    app-icons tar failed: $?\n");
    BSUtil::cleandir("$dir/app-icons");
  }
  rmdir("$dir/app-icons");
  $appdatas->{'version'} ||= '0.6';
  my $rootname = @{$appdatas->{'application'} || []} ? 'applications' : 'components';
  my $appdatasxml = XML::Simple::XMLout($appdatas, 'RootName' => $rootname,  XMLDecl => '<?xml version="1.0" encoding="UTF-8"?>');
  Encode::_utf8_off($appdatasxml);
  writestr("$dir/appdata.xml", undef, $appdatasxml);
}

sub merge_package_appdata {
  my ($appdatas, $bin, $appdataxml) = @_;

  my $appdata;
  eval {
    $appdata = XML::Simple::XMLin($appdataxml, 'ForceArray' => 1, 'KeepRoot' => 1);
  };
  warn("$bin: $@") if $@;
  return $appdatas unless $appdata;

  if ($appdata->{'components'} || $appdata->{'applications'}) {
    # appstream data as it ought to be
    $appdata = $appdata->{'components'} || $appdata->{'applications'};
    $appdata = $appdata->[0] if ref($appdata) eq 'ARRAY';	# XML::Simple is weird
  } elsif ($appdata->{'component'} || $appdata->{'application'}) {
    # bad: just the appdata itself. no version info. assume 0.8 if we have components
    $appdata->{'version'} ||= '0.8' if $appdata->{'component'};
  } else {
    return $appdatas;		# huh?
  }

  # do some basic checking
  return $appdatas unless $appdata && ref($appdata) eq 'HASH';
  return $appdatas unless $appdata->{'component'} || $appdata->{'application'};
  return $appdatas if $appdata->{'component'} && ref($appdata->{'component'}) ne 'ARRAY';
  return $appdatas if $appdata->{'application'} && ref($appdata->{'application'}) ne 'ARRAY';

  # merge the applications/components
  if ($appdatas) {
    if ($appdata->{'version'}) {
      my $v1 = $appdata->{'version'};
      my $v2 = $appdatas->{'version'} || '';
      $v1 =~ s/(\d+)/substr("00000000$1", -9)/ge;
      $v2 =~ s/(\d+)/substr("00000000$1", -9)/ge;
      $appdatas->{'version'} = $appdata->{'version'} if $v1 gt $v2; 
    }
    if ($appdata->{'component'} || $appdatas->{'component'}) {
      $appdatas->{'component'} = delete $appdatas->{'application'} if $appdatas->{'application'};
      push @{$appdatas->{'component'}}, @{$appdata->{'component'} || $appdata->{'application'} || []}; 
    } else {
      push @{$appdatas->{'application'}}, @{$appdata->{'application'} || []}; 
    }
  } else {
    $appdatas = $appdata;
  }
  $appdatas->{'origin'} = 'appdata' if $appdatas->{'version'} && $appdatas->{'version'} >= 0.8; 
  return $appdatas;
}

# repoindex content for the ZYPP service
sub createrepo_zyppservice {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  my $downloadurl = BSUrlmapper::get_downloadurl("$projid/$repoid");
  return unless $downloadurl;

  local *FILE;
  my $projidHeader = $data->{'dbgsplit'} ? "$projid$data->{'dbgsplit'}" : $projid;
  $projidHeader =~ s/:/_/g;

  mkdir_p("$extrep/repo");

  # The repoindex file
  open(FILE, '>', "$extrep/repo/repoindex.xml$$") || die("$extrep/repo/repoindex.xml$$: $!\n");
  print FILE "<?xml version=\"1.0\"?>\n";
  print FILE "<repoindex>\n";
  my $repoinfo = $data->{'repoinfo'};
  my @sprp = @{$repoinfo->{'prpsearchpath'} || []};
  while (@sprp) {
    my $sprp = shift @sprp;
    next unless @sprp; # ignore the last repo, supposed to be the base repo
    my $sprojid = $sprp->{'project'};
    my $srepoid = $sprp->{'repository'};
    my $url = BSUrlmapper::get_downloadurl("$sprojid/$srepoid");
    next unless defined $url;
    print FILE "  <repo alias=\"$sprojid\" url=\"$url\" enabled=\"1\" />\n";
  }
  print FILE "</repoindex>\n";
  close(FILE) || die("close: $!\n");
  rename("$extrep/repo/repoindex.xml$$", "$extrep/repo/repoindex.xml") || die("rename $extrep/repo/repoindex.xml$$ $extrep/repo/repoindex.xml: $!\n");

  # sign it .. even when zypper is not using it atm
  if ($BSConfig::sign) {
    my @signargs;
    push @signargs, '--project', $projid if $BSConfig::sign_project;
    push @signargs, @{$data->{'signargs'} || []};
    qsystem($BSConfig::sign, @signargs, '-d', "$extrep/repo/repoindex.xml") && die("    sign failed: $?\n");
    pickup_detached_signature($data, "$extrep/repo/repoindex.xml");
  }

# Needs further discussion with zypp people
  # The service file
#  open(FILE, '>', "$extrep/zypp.service$$") || die("$extrep/zypp.service$$: $!\n");
#  print FILE "[$projidHeader]\n";
#  print FILE "name=$projid\n";
#  print FILE "type=rsi\n";
#  print FILE "enabled=1\n";
#  print FILE "autorefresh=1\n";
#  print FILE "url=$downloadurl\n";
#  close(FILE) || die("close: $!\n");
#  rename("$extrep/zypp.service$$", "$extrep/zypp.service") || die("rename $extrep/zypp.service$$ $extrep/zypp.service: $!\n");
}

sub deleterepo_zyppservice {
  my ($extrep, $projid, $repoid, $data) = @_;
  # cleanup files
  unlink("$extrep/zypp.service");
  unlink("$extrep/repo/repoindex.xml.asc");
  unlink("$extrep/repo/repoindex.xml");
  rmdir("$extrep/repo");
}

sub createrepo_rpmmd {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  my %options = map {$_ => 1} @{$options || []};
  my @repotags = @{$data->{'repotags'} || []};
  print "    running createrepo\n";
  my $createrepo_bin = $BSConfig::createrepo ? $BSConfig::createrepo : 'createrepo';
  my $modifyrepo_bin = $BSConfig::modifyrepo ? $BSConfig::modifyrepo : 'modifyrepo';
  # cleanup files
  unlink("$extrep/repodata/repomd.xml.asc");
  unlink("$extrep/repodata/repomd.xml.key");
  unlink("$extrep/repodata/latest-feed.xml");
  unlink("$extrep/repodata/index.html");
  my @oldrepodata = ls("$extrep/repodata");
  qsystem('rm', '-rf', "$extrep/repodata/repoview") if -d "$extrep/repodata/repoview";
  qsystem('rm', '-rf', "$extrep/repodata/.olddata") if -d "$extrep/repodata/.olddata";
  qsystem('rm', '-f', "$extrep/repodata/patterns*");
  qsystem('rm', '-rf', "$extrep/.repodata") if -d "$extrep/.repodata";

  # run createrepo
  unlink("$extrep/.chksums");
  if (@{$data->{'chksumfiles'} || []}) {
    local *CHKSUMS;
    if (open(CHKSUMS, '>', "$extrep/.chksums")) {
      for my $chksumfile (@{$data->{'chksumfiles'} || []}) {
        my $chksums = BSUtil::retrieve($chksumfile, 1) || {};
        for my $packid (sort keys %$chksums) {
          print CHKSUMS $chksums->{$packid};
        }
      }
      close CHKSUMS;
      $ENV{'CREATEREPO_CHECKSUMS'} = "$extrep/.chksums";
    }
  }
  my $repoinfo = $data->{'repoinfo'};
  # create generic rpm-md meta data
  my @createrepoargs;
  push @createrepoargs, '--changelog-limit', '20';
  push @createrepoargs, map { ('--repo', $_ ) } @repotags;
  push @createrepoargs, '--content', 'debug' if $data->{'dbgsplit'};
  push @createrepoargs, '--content', 'update' if ($repoinfo->{'projectkind'} || '') eq 'maintenance_incident';
  push @createrepoargs, '--content', 'update' if ($repoinfo->{'projectkind'} || '') eq 'maintenance_release';
  push @createrepoargs, '--filelists-ext' if ($options{'filelists-ext'} || $options{'filelists_ext'});
  my @legacyargs;
  if ($options{'sha512'}) {
    push @legacyargs, '--unique-md-filenames', '--checksum=sha512';
  } elsif ($options{'legacy'}) {
    push @legacyargs, '--simple-md-filenames', '--checksum=sha';
  } else {
    # the default in newer createrepos
    push @legacyargs, '--unique-md-filenames', '--checksum=sha256';
  }
  if ($options{'zchunk'}) {
    # we currently do not support user generated zchunk dictionaries for
    # security reasons.
    for my $sprp (@{$repoinfo->{'prpsearchpath'} || []}) {
      my $zchunk_dict_dir = "$BSConfig::bsdir/zchunk/$sprp->{'project'}";
      next unless -d $zchunk_dict_dir;
      print "    using chunk data of project $sprp->{'project'}\n";
      push @legacyargs, "--zck", "--zck-dict-dir=$zchunk_dict_dir";
      last;
    }
  }
  my @updateargs;
  # createrepo 0.9.9 defaults to creating the sqlite database.
  # We do disable it since it is time and space consuming.
  # doing this via @updateargs for the case that an old createrepo is installed which does not support
  # this switch.
  push @updateargs, '--no-database';
  if (-f "$extrep/repodata/repomd.xml") {
    push @updateargs, '--update';
    # createrepo_c 0.20.0 defaults to keep always additional metadata. We must change
    # to old behaviour or old data would stay forever when it got removed actually
    push @createrepoargs, '--discard-additional-metadata';
  }
  if (qsystem($createrepo_bin, '-q', '-c', "$extrep/repocache", @updateargs, @createrepoargs, @legacyargs, $extrep)) {
    die("    createrepo failed: $?\n") unless @updateargs;
    print("    createrepo failed: $?\n");
    print "    re-running without extra options\n";
    qsystem($createrepo_bin, '-q', '-c', "$extrep/repocache", @createrepoargs, @legacyargs, $extrep) && die("    createrepo failed again: $?\n");
  }
  unlink("$extrep/.chksums");
  if (@{$data->{'chksumfiles'} || []}) {
    delete $ENV{'CREATEREPO_CHECKSUMS'};
  }

  # add updateinfos
  unlink("$extrep/repodata/$_") for grep {/updateinfo\.xml/} @oldrepodata;
  if (@{$data->{'updateinfos'} || []}) {
    print "    adding updateinfo.xml to repodata\n";
    # strip supportstatus and patchinforef from updateinfos
    my $updateinfos = BSUtil::clone($data->{'updateinfos'});
    for my $up (@$updateinfos) {
      delete $up->{'patchinforef'};
      for my $cl (@{($up->{'pkglist'} || {})->{'collection'} || []}) {
	delete $_->{'supportstatus'} for @{$cl->{'package'} || []};
      }
    }
    writexml("$extrep/repodata/updateinfo.xml", undef, {'update' => $updateinfos}, $BSXML::updateinfo);
    qsystem($modifyrepo_bin, "$extrep/repodata/updateinfo.xml", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
    unlink("$extrep/repodata/updateinfo.xml");
  }

  # add appdata
  unlink("$extrep/repodata/$_") for grep {/appdata\.xml/ || /app-icons/ || /appdata-icons/} @oldrepodata;
  if (%{$data->{'appdatas'} || {}}) {
    create_appdata_files("$extrep/repodata", $data->{'appdatas'});
    if (-e "$extrep/repodata/appdata.xml") {
      print "    adding appdata.xml to repodata\n";
      qsystem($modifyrepo_bin, "$extrep/repodata/appdata.xml", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
      unlink("$extrep/repodata/appdata.xml");
    }
    if (-e "$extrep/repodata/app-icons.tar") {
      print "    adding app-icons.tar to repodata\n";
      qsystem($modifyrepo_bin, "$extrep/repodata/app-icons.tar", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
      rename("$extrep/repodata/app-icons.tar", "$extrep/repodata/appdata-icons.tar");
      print "    adding appdata-icons.tar to repodata\n";
      qsystem($modifyrepo_bin, "$extrep/repodata/appdata-icons.tar", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
      unlink("$extrep/repodata/appdata-icons.tar");
    }
  }

  # add deltainfo
  unlink("$extrep/repodata/$_") for grep {/(?:deltainfo|prestodelta)\.xml/} @oldrepodata;
  if (%{$data->{'deltainfos'} || {}} && ($options{'deltainfo'} || $options{'prestodelta'})) {
    print "    adding deltainfo.xml to repodata\n" if $options{'deltainfo'};
    print "    adding prestodelta.xml to repodata\n" if $options{'prestodelta'};
    # things are a bit complex, as we have to merge the deltas, and we also have to add the checksum
    my %mergeddeltas;
    for my $d (values(%{$data->{'deltainfos'}})) {
      my $checksum = 'sha256';
      $checksum = 'sha' if $options{'legacy'};
      $checksum = 'sha512' if $options{'sha512'};
      addsizechecksum("$extrep/$d->{'delta'}->[0]->{'filename'}", $d->{'delta'}->[0], $checksum);
      my $mkey = "$d->{'arch'}\0$d->{'name'}\0$d->{'epoch'}\0$d->{'version'}\0$d->{'release'}\0";
      if ($mergeddeltas{$mkey}) {
	push @{$mergeddeltas{$mkey}->{'delta'}}, $d->{'delta'}->[0];
      } else {
	$mergeddeltas{$mkey} = $d;
      }
    }
    # got all, now write
    my @mergeddeltas = map {$mergeddeltas{$_}} sort keys %mergeddeltas;
    if ($options{'deltainfo'}) {
      writexml("$extrep/repodata/deltainfo.xml", undef, {'newpackage' => \@mergeddeltas}, $BSXML::deltainfo);
      qsystem($modifyrepo_bin, "$extrep/repodata/deltainfo.xml", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
      unlink("$extrep/repodata/deltainfo.xml");
    }
    if ($options{'prestodelta'}) {
      writexml("$extrep/repodata/prestodelta.xml", undef, {'newpackage' => \@mergeddeltas}, $BSXML::prestodelta);
      qsystem($modifyrepo_bin, "$extrep/repodata/prestodelta.xml", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
      unlink("$extrep/repodata/prestodelta.xml");
    }
  }

  # add modulemd data
  unlink("$extrep/repodata/modules.yaml");
  unlink("$extrep/repodata/modules-unfiltered.yaml");
  if ($data->{'modulemds'}) {
    writestr("$extrep/repodata/modules-unfiltered.yaml", undef, $data->{'modulemds'});
    # unpack primary xml file
    qsystem('chdir', "$extrep/repodata", 'stdout', 'modules.yaml', "$INC[0]/build/writemodulemd", '--filter', 'modules-unfiltered.yaml', '.') && die("    writemodulemd failed: $?\n");
    unlink("$extrep/repodata/modules-unfiltered.yaml");
    if (-s "$extrep/repodata/modules.yaml") {
      qsystem($modifyrepo_bin, "$extrep/repodata/modules.yaml", "$extrep/repodata", @legacyargs) && die("    modifyrepo failed: $?\n");
    }
    unlink("$extrep/repodata/modules.yaml");
  }
  

  # prune repocache
  if (-d "$extrep/repocache") {
    my $now = time;
    for (map { "$extrep/repocache/$_" } ls("$extrep/repocache")) {
      my @s = stat($_);
      unlink($_) if @s && $s[9] < $now - 7*86400;
    }
  }

  my $title = $data->{'repoinfo'}->{'title'};
  my $downloadurl = BSUrlmapper::get_downloadurl("$projid/$repoid");
  if (-x "/usr/bin/repoview") {
    my @downloadurlarg;
    @downloadurlarg = ("-u$downloadurl") if $downloadurl;
    print "    running repoview\n";
    qsystem('repoview', '-f', @downloadurlarg, "-t$title", $extrep) && print("    repoview failed: $?\n");
  }

  if ($BSConfig::createrepo_rpmmd_hook) {
    $BSConfig::createrepo_rpmmd_hook->($projid, $repoid, $extrep, \%options, $data);
  }

  if ($options{'rsyncable'}) {
    if (-x '/usr/bin/rezip_repo_rsyncable') {
      print "    re-compressing metadata with --rsyncable\n";
      unlink("$extrep/repodata/repomd.xml.asc");
      qsystem('/usr/bin/rezip_repo_rsyncable', $extrep) && print("    rezip_repo_rsyncable failed: $?\n");
    } else {
      print "    /usr/bin/rezip_repo_rsyncable not installed, ignoring the rsyncable option\n";
    }
  }

  if ($BSConfig::sign && -e "$extrep/repodata/repomd.xml") {
    my @signargs;
    push @signargs, '--project', $projid if $BSConfig::sign_project;
    push @signargs, @{$data->{'signargs'} || []};
    qsystem($BSConfig::sign, @signargs, '-d', "$extrep/repodata/repomd.xml") && die("    sign failed: $?\n");
    writestr("$extrep/repodata/repomd.xml.key", undef, $data->{'pubkey'}) if $data->{'pubkey'};
    pickup_detached_signature($data, "$extrep/repodata/repomd.xml");
  }
  if ($downloadurl) {
    local *FILE;
    open(FILE, '>', "$extrep/$projid.repo$$") || die("$extrep/$projid.repo$$: $!\n");
    my $projidHeader = $data->{'dbgsplit'} ? "$projid$data->{'dbgsplit'}" : $projid;
    $projidHeader =~ s/:/_/g;
    print FILE "[$projidHeader]\n";
    print FILE "name=$title\n";
    print FILE "type=rpm-md\n";
    print FILE "baseurl=$downloadurl\n";
    if ($BSConfig::sign) {
      print FILE "gpgcheck=1\n";
      if (-e "$extrep/repodata/repomd.xml.key") {
        print FILE "gpgkey=${downloadurl}repodata/repomd.xml.key\n";
      } else {
        die("neither a project key is available nor gpg_standard_key is set\n") unless defined($BSConfig::gpg_standard_key);
        print FILE "gpgkey=$BSConfig::gpg_standard_key\n";
      }
    }
    print FILE "enabled=1\n";
    close(FILE) || die("close: $!\n");
    rename("$extrep/$projid.repo$$", "$extrep/$projid.repo") || die("rename $extrep/$projid.repo$$ $extrep/$projid.repo: $!\n");
  }
}

sub deleterepo_rpmmd {
  my ($extrep, $projid) = @_;

  qsystem('rm', '-rf', "$extrep/repodata") if -d "$extrep/repodata";
  qsystem('rm', '-rf', "$extrep/.repodata") if -d "$extrep/.repodata";
  unlink("$extrep/$projid.repo");
}

sub createrepo_virtbuilder {
  my ($extrep, $projid, $repoid, $data) = @_;

  # cleanup
  unlink("$extrep/index.key");
  unlink("$extrep/index.asc");

  # Sign the index
  if ($BSConfig::sign && -e "$extrep/index") {
    print "Signing the index for $projid/$repoid\n";
    my @signargs;
    push @signargs, '--project', $projid if $BSConfig::sign_project;
    push @signargs, @{$data->{'signargs'} || []};
    print "Running command: $BSConfig::sign @signargs -c $extrep/index\n";
    qsystem($BSConfig::sign, @signargs, '-c', "$extrep/index") && die("    sign failed: $?\n");
    writestr("$extrep/index.key", undef, $data->{'pubkey'}) if $data->{'pubkey'};
  }
}

sub createrepo_helm {
  my ($extrep, $projid, $repoid, $data, $options, $containers) = @_;
  print "    generating helm chart index\n";
  my $downloadurl = BSUrlmapper::get_downloadurl("$projid/$repoid");
  unlink("$extrep/index.yaml");
  my %entries;
  for my $p (sort keys %$containers) {
    next unless ($containers->{$p}->{'type'} || '') eq 'helm';
    my $helminfo = $containers->{$p};
    my $entry = BSPublisher::Helm::mkindexentry($helminfo, $downloadurl ? "${downloadurl}$p" : undef);
    push @{$entries{$helminfo->{'name'}}}, $entry if $entry;
  }
  my $index_yaml = BSPublisher::Helm::mkindex_yaml(\%entries);
  writestr("$extrep/index.yaml", undef, $index_yaml);
}

sub deleterepo_helm {
  my ($extrep, $projid) = @_;
  unlink("$extrep/index.yaml");
}

sub createrepo_hdlist2 {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  print "    running hdlist2\n";
  # create generic rpm-md meta data
  for my $arch (ls($extrep)) {
    next if $arch =~ /^\./;
    my $r = "$extrep/$arch";
    next unless -d $r;
    if (qsystem('genhdlist2', '--allow-empty-media', $r)) {
      print("    genhdlist2 failed: $?\n");
    }
  }
  # signing is done only via rpm packages to my information
}

sub deleterepo_hdlist2 {
  my ($extrep, $projid) = @_;

  for my $arch (ls($extrep)) {
    next if $arch =~ /^\./;
    my $r = "$extrep/$arch";
    next unless -d $r;
    qsystem('rm', '-rf', "$r/media_info") if -d "$r/media_info";
  }
}

sub createrepo_susetags {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  mkdir_p("$extrep/media.1");
  mkdir_p("$extrep/descr");
  my @lt = localtime(time());
  $lt[4] += 1;
  $lt[5] += 1900;
  my $str = sprintf("Open Build Service\n%04d%02d%02d%02d%02d%02d\n1\n", @lt[5,4,3,2,1,0]);
  writestr("$extrep/media.1/.media", "$extrep/media.1/media", $str);
  writestr("$extrep/media.1/.directory.yast", "$extrep/media.1/directory.yast", "media\n");
  $str = <<"EOL";
PRODUCT Open Build Service $projid $repoid
VERSION 1.0-0
LABEL $data->{'repoinfo'}->{'title'}
VENDOR Open Build Service
ARCH.x86_64 x86_64 i686 i586 i486 i386 noarch
ARCH.k1om k1om noarch
ARCH.ppc64p7 ppc64p7 noarch
ARCH.ppc64 ppc64 ppc noarch
ARCH.ppc64le ppc64le noarch
ARCH.ppc ppc noarch
ARCH.riscv64 riscv64 noarch
ARCH.sh4 sh4 noarch
ARCH.m68k m68k noarch
ARCH.aarch64 aarch64 aarch64_ilp32 noarch
ARCH.aarch64_ilp32 aarch64_ilp32 noarch
ARCH.armv4l arm       armv4l noarch
ARCH.armv5l arm armel armv4l armv5l armv5tel noarch
ARCH.armv6l arm armel armv4l armv5l armv5tel armv6l armv6vl armv6hl noarch
ARCH.armv7l arm armel armv4l armv5l armv5tel armv6l armv6vl armv7l armv7hl noarch
ARCH.i686 i686 i586 i486 i386 noarch
ARCH.i586 i586 i486 i386 noarch
DEFAULTBASE i586
DESCRDIR descr
DATADIR .
EOL
  writestr("$extrep/.content", "$extrep/content", $str);
  print "    running create_package_descr\n";
  qsystem('chdir', $extrep, 'create_package_descr', '-o', 'descr', '-x', '/dev/null') && print "    create_package_descr failed: $?\n";
  unlink("$extrep/descr/directory.yast");
  my @d = map {"$_\n"} sort(ls("$extrep/descr"));
  writestr("$extrep/descr/.directory.yast", "$extrep/descr/directory.yast", join('', @d));
}

sub deleterepo_susetags {
  my ($extrep) = @_;

  unlink("$extrep/directory.yast");
  unlink("$extrep/content");
  unlink("$extrep/media.1/media");
  unlink("$extrep/media.1/directory.yast");
  rmdir("$extrep/media.1");
  qsystem('rm', '-rf', "$extrep/descr") if -d "$extrep/descr";
}

sub compress_and_rename {
  my ($tmpfile, $file) =@_;
  if (-s $tmpfile) {
    unlink($file);
    link($tmpfile, $file);
    qsystem('gzip', '-9', '-n', '-f', $tmpfile) && print "    gzip $tmpfile failed: $?\n";
    unlink($tmpfile);
    unlink("$file.gz");
    rename("$tmpfile.gz", "$file.gz");
  } else {
    unlink($tmpfile);
    unlink($file);
    unlink("$file.gz");
  }
}

sub fixup_scanpackages_output {
  my ($file) = @_;
  return unless -s $file;
  my $f;
  open ($f, '<', $file) || die("$file: $!\n");
  my $f2;
  open ($f2, '>', "$file.tmp") || die("$file.tmp: $!\n");
  while (<$f>) {
    s/^Filename: \.\//Filename: /;
    print $f2 $_ or die("$file.tmp: $!\n");
  }
  close($f);
  close($f2) || die("$file.tmp: $!\n");
  rename("$file.tmp", $file) || die("rename $file.tmp $file: $!\n");
}

sub createrepo_debian {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  print "    running dpkg-scanpackages\n";
  if (qsystem('chdir', $extrep, 'stdout', 'Packages.new', 'dpkg-scanpackages', '-m', '.', '/dev/null')) {
    die("    dpkg-scanpackages failed: $?\n");
  }
  fixup_scanpackages_output("$extrep/Packages.new");
  compress_and_rename("$extrep/Packages.new", "$extrep/Packages");

  print "    running dpkg-scansources\n";
  if (qsystem('chdir', $extrep, 'stdout', 'Sources.new', 'dpkg-scansources', '.', '/dev/null')) {
    die("    dpkg-scansources failed: $?\n");
  }
  compress_and_rename("$extrep/Sources.new", "$extrep/Sources");
  createrelease_debian($extrep, $projid, $repoid, $data, $options);

  my $udebs = "$extrep/debian-installer";
  mkdir_p($udebs) unless -d $udebs;
  if (qsystem('chdir', $udebs, 'stdout', 'Packages.new', 'dpkg-scanpackages', '-t', 'udeb', '-m', '..', '/dev/null')) {
    die("    dpkg-scanpackages for udebs failed: $?\n");
  }
  fixup_scanpackages_output("$udebs/Packages.new");
  compress_and_rename("$udebs/Packages.new", "$udebs/Packages");

  if ( -e "$udebs/Packages") {
    createrelease_debian($udebs, $projid, $repoid, $data, $options);
  } else {
    rmdir($udebs);
  }
}

sub createrelease_debian {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  my $obsname = $BSConfig::obsname || 'build.opensuse.org';
  my $date = POSIX::asctime(gmtime(time()));
  chomp $date;
  my @debarchs = map {Build::Deb::basearch($_)} @{$data->{'repoinfo'}->{'arch'} || []};
  my $archs = join(' ', @debarchs);

  # The Release file enables users to use Pinning. See also:
  #  http://www.debian.org/doc/manuals/repository-howto/repository-howto#release
  #  apt_preferences(5)
  #
  # Note:
  # There is no Version because this is not part of a Debian release (yet).
  # The Component line is missing to not accidently associate the packages with
  # a Debian licensing component.
  my $str = <<"EOL";
Archive: $repoid
Codename: $repoid
Origin: obs://$obsname/$projid/$repoid
Label: $projid
Architectures: $archs
Date: $date
Description: $data->{'repoinfo'}->{'title'}
MD5Sum:
EOL

  open(OUT, '>', "$extrep/Release") || die("$extrep/Release: $!\n");
  print OUT $str;
  close(OUT) || die("close: $!\n");

  # append checksums
  my $sha1sums = "SHA1:\n";
  my $sha256sums = "SHA256:\n";
  open(OUT, '>>', "$extrep/Release") || die("$extrep/Release: $!\n");
  for my $f ( "Packages", "Packages.gz", "Sources", "Sources.gz" ) {
    my @s = stat("$extrep/$f");
    next unless @s;
    my $fdata = readstr("$extrep/$f");
    my $md5  = Digest::MD5::md5_hex($fdata);
    my $size = $s[7];
    print OUT " $md5 $size $f\n";
    my $sha1  = Digest::SHA::sha1_hex($fdata);
    $sha1sums .= " $sha1 $size $f\n";
    my $sha256  = Digest::SHA::sha256_hex($fdata);
    $sha256sums .= " $sha256 $size $f\n";
  }
  print OUT $sha1sums;
  print OUT $sha256sums;
  close(OUT) || die("close: $!\n");

  unlink("$extrep/Release.gpg");
  unlink("$extrep/Release.key");

  # re-sign changed Release file
  if ($BSConfig::sign && -e "$extrep/Release") {
    my @signargs;
    push @signargs, '--project', $projid if $BSConfig::sign_project;
    push @signargs, @{$data->{'signargs'} || []};
    qsystem($BSConfig::sign, @signargs, '-d', "$extrep/Release") && die("    sign failed: $?\n");
    pickup_detached_signature($data, "$extrep/Release");
    rename("$extrep/Release.asc","$extrep/Release.gpg");
    BSUtil::cp("$extrep/Release", "$extrep/InRelease.tmp");
    qsystem($BSConfig::sign, @signargs, '-c', "$extrep/InRelease.tmp") && die("    sign failed: $?\n");
    rename("$extrep/InRelease.tmp","$extrep/InRelease");
  }
  if ($BSConfig::sign) {
    writestr("$extrep/Release.key", undef, $data->{'pubkey'}) if $data->{'pubkey'};
  }
}

sub deleterepo_debian {
  my ($extrep) = @_;

  unlink("$extrep/Packages");
  unlink("$extrep/Packages.gz");
  unlink("$extrep/Sources");
  unlink("$extrep/Sources.gz");
  unlink("$extrep/Release");
  unlink("$extrep/Release.gpg");
  unlink("$extrep/Release.key");
  if (-d "$extrep/debian-installer") {
    BSUtil::cleandir("$extrep/debian-installer");
    rmdir("$extrep/debian-installer");
  }
}


##########################################################################

sub createrepo_arch {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  deleterepo_arch($extrep);
  my $rname = $projid;
  $rname .= "_$repoid" if $repoid ne 'standard';
  $rname =~ s/:/_/g;
  for my $arch (ls($extrep)) {
    next unless -d "$extrep/$arch";
    print "    running bs_mkarchrepo $arch\n";
    qsystem("$INC[0]/bs_mkarchrepo", $rname, "$extrep/$arch") && die("    repo creation failed: $?\n");
    if (-e "$extrep/$arch/$rname.db.tar.gz") {
      link("$extrep/$arch/$rname.db.tar.gz", "$extrep/$arch/$rname.db");
    }
    if (-e "$extrep/$arch/$rname.files.tar.gz") {
      link("$extrep/$arch/$rname.files.tar.gz", "$extrep/$arch/$rname.files");
    }
    if ($BSConfig::sign) {
      my @signargs;
      push @signargs, '--project', $projid if $BSConfig::sign_project;
      push @signargs, @{$data->{'signargs'} || []};
      if (-e "$extrep/$arch/$rname.db.tar.gz") {
        qsystem($BSConfig::sign, @signargs, '-D', "$extrep/$arch/$rname.db.tar.gz") && die("    sign failed: $?\n");
        link("$extrep/$arch/$rname.db.tar.gz.sig", "$extrep/$arch/$rname.db.sig");
      }
      if (-e "$extrep/$arch/$rname.files.tar.gz") {
        qsystem($BSConfig::sign, @signargs, '-D', "$extrep/$arch/$rname.files.tar.gz") && die("    sign failed: $?\n");
        link("$extrep/$arch/$rname.files.tar.gz.sig", "$extrep/$arch/$rname.files.sig");
      }
      writestr("$extrep/$arch/$rname.key", undef, $data->{'pubkey'}) if $data->{'pubkey'};
    }
  }
}

sub deleterepo_arch {
  my ($extrep) = @_;
  for my $arch (ls($extrep)) {
    next unless -d "$extrep/$arch";
    next if $arch eq 'repodata' || $arch eq 'repocache' || $arch eq 'media.1' || $arch eq 'descr';
    for (grep {/\.(?:\?db|db\.tar\.gz|files|files\.tar\.gz|key)(?:\.sig)?$/} ls("$extrep/$arch")) {
      unlink("$extrep/$arch/$_");
    }
  }
}

##########################################################################

sub createrepo_staticlinks {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  my $versioned = grep {$_ eq 'versioned'} @{$options || []};
  for my $arch ('.', ls($extrep)) {
    next unless -d "$extrep/$arch";
    for (ls("$extrep/$arch")) {
      if (-l "$extrep/$arch/$_") {
        unlink "$extrep/$arch/$_" unless -e "$extrep/$arch/$_";
        next;
      }
      my $link;
      if (/^(.*)-Build(?:\d\d\d\d+|\d+\.\d+)(-Media\d?(\.license)?)$/s) {
        $link = "$1$2"; # no support for versioned links
      }
      if (/^(.*)-([^-]*)-[^-]*\.rpm$/s) {
        $link = "$1.rpm";
        $link = "$1-$2.rpm" if $versioned;
      } elsif (/^(.*)_([^_]*)-[^_]*\.([ud]?deb)$/s) {
        $link = "$1.$3";
        $link = "${1}_$2.$3" if $versioned;
      } elsif (/^(.*)-([^-]*)-([^-]*)-([^-]*)\.(AppImage?(\.zsync)?)$/s) {
        # name version "release.glibcX.Y" arch suffix
        $link = "$1-latest-$4.$5";
      } elsif (/^(.*)_([^_]*)_([^_]*)-Build[^_]*\.snap$/s) {
        $link = "${1}_${3}.snap";
        $link = "${1}_${3}_$2.snap" if $versioned;
      } elsif (/^(.*)-Build(?:\d\d\d\d+|\d+(?:\.\d+)+)(-Media\d?)\.iso$/s) {
        # product builds
        $link = "$1$2.iso"; # no support for versioned links
      } elsif (/^(.*\.(?:$binarchs)?)-([^-]+)?(-.*?)?-([^-]+)(\.(raw\.install\.raw\.xz|raw\.xz|tar\.xz|box|json|install\.iso|install\.tar|tbz|tgz|vmx|vmdk|vmdk\.xz|appx|vhdx|vhdx\.xz|vdi|vdi\.xz|vhdfixed\.xz|iso|phar|qcow2|qcow2\.xz|ova|ova\.xz|cpio\.tar\.bz2))$/s) {
        # kiwi appliance
        my $profile = $3 || "";
        $link = "$1$profile$5";
        $link = "$1-$2$profile$5" if $versioned;
      }
      next unless $link;
      unlink("$extrep/$arch/.$link"); # drop left over
      symlink($_, "$extrep/$arch/.$link");
      rename("$extrep/$arch/.$link", "$extrep/$arch/$link"); # atomic update

      # check for detached sha256 files
      if ( -e "$extrep/$arch/$_.sha256" ) {
        # copy and patch the filename
        my ($econtent, $ehadsig) = depgp(readstr("$extrep/$arch/$_.sha256"));
        $econtent =~ s/([ \/])\Q$_\E\n/$link\n/g;
        unlink("$extrep/$arch/$link.sha256");
        writestr("$extrep/$arch/$link.sha256", undef, $econtent);
      }
      if ( -e "$extrep/$arch/$_.sha256.asc" ) {
        # regenerate signature
        if ($BSConfig::sign && -e "$extrep/$arch/$link.sha256") {
          my @signargs;
          push @signargs, '--project', $projid if $BSConfig::sign_project;
          push @signargs, @{$data->{'signargs'} || []};
          qsystem($BSConfig::sign, @signargs, '-d', "$extrep/$arch/$link.sha256") && die("    sign failed: $?\n");
          pickup_detached_signature($data, "$extrep/$arch/$link.sha256");
        }
      }
    }
  }
}

sub deleterepo_staticlinks {
  my ($extrep) = @_;
  for my $arch ('.', ls($extrep)) {
    next unless -d "$extrep/$arch";
    for (ls("$extrep/$arch")) {
      next unless -l "$extrep/$arch/$_";
      next if /\.(?:db|files)(?:\.sig)?$/;
      unlink("$extrep/$arch/$_");
    }
  }
}

##########################################################################

sub createrepo_vagrant {
  my ($extrep, $projid, $repoid, $data, $options) = @_;
  my %boxes;
  my $withversion = grep {$_ eq 'withversion'} @{$options || []};
  for my $box (grep {/^[^\.].*\.box$/} sort(ls($extrep))) {
    next if -l "$extrep/$box"; # skip symlinks
    # kiwi uses the following format:
    # name.arch-verson.format_name
    #   OBS appends "[profile-]BuildXXX" to the version
    #   format_name is "vagrant.<provider>.box" for vagrant builds
    my $xbox = $box;
    # split off format_name with provider
    next unless $xbox =~ s/\.vagrant\.([^\.]+)\.box//;
    my $provider = $1;
    # split off release
    next unless $xbox =~ s/-(?:Build|Snapshot)([^-]*)$//;
    my $release = $1;
    if (!$withversion) {
      # split off profile
      my $profile = '';
      $profile = $1 if $xbox =~ s/((?:-[^0-9-][^-]*)+)$//;
      # split off version
      next unless $xbox =~ s/-([0-9][^-]*)$//;
      my $version = $1;
      $profile = '' if $profile eq "-$provider";
      # add to boxes (last one wins)
      $boxes{"$xbox$profile"}->{"$version.$release"}->{$provider} = $box;
    } else {
      # strip profile if it is the same as the provider
      $xbox =~ s/-\Q$provider\E$//;
      # add to boxes (last one wins)
      $boxes{$xbox}->{$release}->{$provider} = $box;
    }
  }
  my $downloadurl = BSUrlmapper::get_downloadurl("$projid/$repoid");
  if (!%boxes || !$downloadurl) {
    deleterepo_vagrant($extrep);
    return;
  }
  # write json
  mkdir_p("$extrep/boxes.new");
  BSUtil::cleandir("$extrep/boxes.new");
  for my $name (sort keys %boxes) {
    my $json = { 'description' => "$name Vagrant Box", 'short_description' => "$name Vagrant Box", 'name' => $name };
    my $boxversions = $boxes{$name};
    my @versions;
    for my $version (sort {0 - Build::Rpm::verscmp($a, $b)} keys %{$boxversions}) {
      my $boxproviders = $boxes{$name}->{$version};
      my @providers;
      for my $provider (sort keys %$boxproviders) {
	my $url = "${downloadurl}$boxproviders->{$provider}";
	push @providers, { 'name' => $provider, 'url' => $url };
      }
      my $description = "$name Vagrant images";
      my $description_html = $description;
      $description_html =~ s/\&/&amp;/g;
      $description_html =~ s/\</&lt;/g;
      $description_html =~ s/\>/&gt;/g;
      $description_html =~ s/\"/&quot;/g;
      push @versions, {
	'version' => $version,
	'status' => 'active',
	'description_html' => "<p>$description_html</p>\n",
	'description_markdown' => "$description\n",
	'providers' => \@providers,
      };
    }
    $json->{'versions'} = \@versions;
    $json = JSON::XS->new->utf8->canonical->pretty->encode($json);
    writestr("$extrep/boxes.new/$name.json", undef, $json);
  }
  # commit
  BSUtil::cleandir("$extrep/boxes.old");
  rmdir("$extrep/boxes.old");
  rename("$extrep/boxes", "$extrep/boxes.old");
  rename("$extrep/boxes.new", "$extrep/boxes");
  BSUtil::cleandir("$extrep/boxes.old");
  rmdir("$extrep/boxes.old");
  BSUtil::cleandir("$extrep/boxes.new");
  rmdir("$extrep/boxes.new");
}

sub deleterepo_vagrant {
  my ($extrep) = @_;
  if (-d "$extrep/boxes") {
    BSUtil::cleandir("$extrep/boxes");
    rmdir("$extrep/boxes");
  }
}

##########################################################################

sub createrepo_checksumsfile {
  my ($extrep, $projid, $repoid, $data, $options) = @_;
  my @sha256sums;
  for my $file (grep {/^[^\.].*\.sha256$/} sort(ls($extrep))) {
    next unless -s "$extrep/$file";
    next if -s _ > 65536;
    my $content = readstr("$extrep/$file", 1);
    next unless $content;
    for my $line (split("\n", $content)) {
      chomp $line;
      push @sha256sums, $line if $line =~ /^[a-f0-9]{64}  .+$/;
    }
  }
  unlink("$extrep/SHA256SUMS");
  unlink("$extrep/SHA256SUMS.asc");
  unlink("$extrep/SHA256SUMS.gpg");
  if (@sha256sums) {
    @sha256sums = sort {substr($a, 66) cmp substr($b, 66)} @sha256sums;
    my $sha256sums = join("\n", @sha256sums) . "\n";
    writestr("$extrep/.SHA256SUMS", "$extrep/SHA256SUMS", $sha256sums);
    if ($BSConfig::sign) {
      my @signargs;
      push @signargs, '--project', $projid if $BSConfig::sign_project;
      push @signargs, @{$data->{'signargs'} || []};
      my %options = map {$_ => 1} @{$options || []};
      if ($options{'rawsig'}) {
	unlink("$extrep/SHA256SUMS.sig");
	qsystem($BSConfig::sign, @signargs, '-D', "$extrep/SHA256SUMS") && die("    sign failed: $?\n");
	rename("$extrep/SHA256SUMS.sig", "$extrep/SHA256SUMS.gpg");
      }
      qsystem($BSConfig::sign, @signargs, '-d', "$extrep/SHA256SUMS") && die("    sign failed: $?\n");
      pickup_detached_signature($data, "$extrep/SHA256SUMS");
    }
  }
}

sub deleterepo_checksumsfile {
  my ($extrep) = @_;
  if (-e "$extrep/SHA256SUMS") {
    unlink("$extrep/SHA256SUMS");
    unlink("$extrep/SHA256SUMS.asc");
    unlink("$extrep/SHA256SUMS.gpg");
  }
}

##########################################################################

sub createpatterns_rpmmd {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  deletepatterns_rpmmd($extrep);
  my $patterns = $data->{'patterns'};
  return unless @{$patterns || []};

  my $modifyrepo_bin = $BSConfig::modifyrepo ? $BSConfig::modifyrepo : 'modifyrepo';

  # create patterns data structure
  my @pats;
  for my $pattern (@$patterns) {
    push @pats, BSUtil::fromxml($pattern->{'data'}, $BSXML::pattern);
  }
  print "    adding patterns to repodata\n";
  my $pats = {'pattern' => \@pats, 'count' => scalar(@pats)};
  writexml("$extrep/repodata/patterns.xml", undef, $pats, $BSXML::patterns);
  my @legacyargs;
  my %options = map {$_ => 1} @{$options || []};
  if ($options{'sha512'}) {
    push @legacyargs, '--unique-md-filenames', '--checksum=sha512';
  } elsif ($options{'legacy'}) {
    push @legacyargs, '--simple-md-filenames', '--checksum=sha';
  } else {
    # the default in newer createrepos
    push @legacyargs, '--unique-md-filenames', '--checksum=sha256';
  }
  qsystem($modifyrepo_bin, "$extrep/repodata/patterns.xml", "$extrep/repodata", @legacyargs) && print("    modifyrepo failed: $?\n");
  unlink("$extrep/repodata/patterns.xml");

#  for my $pattern (@{$patterns || []}) {
#    my $pname = "patterns.$pattern->{'name'}";
#    $pname =~ s/\.xml$//;
#    print "    adding pattern $pattern->{'name'} to repodata\n";
#    writestr("$extrep/repodata/$pname.xml", undef, $pattern->{'data'});
#    qsystem('modifyrepo', "$extrep/repodata/$pname.xml", "$extrep/repodata", @legacyargs) && print("    modifyrepo failed: $?\n");
#    unlink("$extrep/repodata/$pname.xml");
#  }

  # re-sign changed repomd.xml file
  if ($BSConfig::sign && -e "$extrep/repodata/repomd.xml") {
    my @signargs;
    push @signargs, '--project', $projid if $BSConfig::sign_project;
    push @signargs, @{$data->{'signargs'} || []};
    qsystem($BSConfig::sign, @signargs, '-d', "$extrep/repodata/repomd.xml") && die("    sign failed: $?\n");
    pickup_detached_signature($data, "$extrep/repodata/repomd.xml");
  }
}

sub deletepatterns_rpmmd {
  my ($extrep) = @_;
  for my $pat (ls("$extrep/repodata")) {
    next unless $pat =~ /^patterns/;
    unlink("$extrep/repodata/$pat");
  }
}

sub createpatterns_comps {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  deletepatterns_comps($extrep);
  my $patterns = $data->{'patterns'};
  return unless @{$patterns || []};

  my $modifyrepo_bin = $BSConfig::modifyrepo ? $BSConfig::modifyrepo : 'modifyrepo';

  # create comps data structure
  my @grps;
  for my $pattern (@$patterns) {
    my $pat = BSUtil::fromxml($pattern->{'data'}, $BSXML::pattern);
    my $grp = { 'id' => $pattern->{'name'} };
    for (@{$pat->{'summary'}}) {
      my $el = { '_content' => $_->{'_content'} };
      $el->{'xml:lang'} = $_->{lang} if $_->{'lang'};
      push @{$grp->{'name'}}, $el;
    }
    for (@{$pat->{'description'}}) {
      my $el = { '_content' => $_->{'_content'} };
      $el->{'xml:lang'} = $_->{'lang'} if $_->{'lang'};
      push @{$grp->{'description'}}, $el;
    }
    for (@{$pat->{'rpm:requires'}->{'rpm:entry'}}) {
      push @{$grp->{'packagelist'}->{'packagereq'} }, { '_content' => $_->{'name'}, 'type' => 'mandatory' };
    }
    for (@{$pat->{'rpm:recommends'}->{'rpm:entry'}}) {
      push @{$grp->{'packagelist'}->{'packagereq'}},  { '_content' => $_->{'name'}, 'type' => 'default' };
    }
    for (@{$pat->{'rpm:suggests'}->{'rpm:entry'}}) {
      push @{$grp->{'packagelist'}->{'packagereq'}},  { '_content' => $_->{'name'}, 'type' => 'optional' };
    }
    push @grps, $grp;
  }
  print "    adding comps to repodata\n";
  my $comps = {'group' => \@grps};
  writexml("$extrep/repodata/group.xml", undef, $comps, $BSXML::comps);
  my @legacyargs;
  my %options = map {$_ => 1} @{$options || []};
  if ($options{'sha512'}) {
    push @legacyargs, '--unique-md-filenames', '--checksum=sha512';
  } elsif ($options{'legacy'}) {
    push @legacyargs, '--simple-md-filenames', '--checksum=sha';
  } else {
    # the default in newer createrepos
    push @legacyargs, '--unique-md-filenames', '--checksum=sha256';
  }
  qsystem($modifyrepo_bin, "$extrep/repodata/group.xml", "$extrep/repodata", @legacyargs) && print("    modifyrepo failed: $?\n");
  unlink("$extrep/repodata/group.xml");

  # re-sign changed repomd.xml file
  if ($BSConfig::sign && -e "$extrep/repodata/repomd.xml") {
    my @signargs;
    push @signargs, '--project', $projid if $BSConfig::sign_project;
    push @signargs, @{$data->{'signargs'} || []};
    qsystem($BSConfig::sign, @signargs, '-d', "$extrep/repodata/repomd.xml") && die("    sign failed: $?\n");
    pickup_detached_signature($data, "$extrep/repodata/repomd.xml");
  }
}

sub deletepatterns_comps {
  my ($extrep) = @_;
  for my $pat (ls("$extrep/repodata")) {
    next unless $pat =~ /group.xml/;
    unlink("$extrep/repodata/$pat");
  }
}


sub createpatterns_ymp {
  my ($extrep, $projid, $repoid, $data, $options) = @_;

  deletepatterns_ymp($extrep, $projid, $repoid);
  my $patterns = $data->{'patterns'};
  return unless @{$patterns || []};

  my $prp_ext = "$projid/$repoid";
  $prp_ext =~ s/:/:\//g;
  my $patterndb = db_open('pattern');

  # get title/description data for all involved projects
  my $repoinfo = $data->{'repoinfo'};
  my %nprojpack;
  my @nprojids = map {$_->{'project'}} @{$repoinfo->{'prpsearchpath'} || []};
  if (@nprojids) {
    my @args = map {"project=$_"} @nprojids;
    my $nprojpack = BSRPC::rpc("$BSConfig::srcserver/getprojpack", $BSXML::projpack, 'nopackages', @args);
    %nprojpack = map {$_->{'name'} => $_} @{$nprojpack->{'project'} || []};
  }

  # get ymp distversion list from config
  my @ympdist;
  for (@{$data->{'config'}->{'publishflags'} || []}) {
    push @ympdist, BSHTTP::urldecode($1) if /^ympdist:(.*)$/s;
  }
  @ympdist = BSUtil::unify(@ympdist) if @ympdist;

  for my $pattern (@$patterns) {
    my $ympname = $pattern->{'name'};
    $ympname =~ s/\.xml$//;
    $ympname .= ".ymp";
    my $pat = BSUtil::fromxml($pattern->{'data'}, $BSXML::pattern);
    next if !exists $pat->{'uservisible'};
    print "    writing ymp for pattern $pat->{'name'}\n";
    my $ymp = {};
    $ymp->{'xmlns:os'} = 'http://opensuse.org/Standards/One_Click_Install';
    $ymp->{'xmlns'} = 'http://opensuse.org/Standards/One_Click_Install';

    my $group = {};
    $group->{'name'} = $pat->{'name'};
    if ($pat->{'summary'}) {
      $group->{'summary'} = $pat->{'summary'}->[0]->{'_content'};
    }
    if ($pat->{'description'}) {
      $group->{'description'} = $pat->{'description'}->[0]->{'_content'};
    }
    my @repos;
    my @sprp = @{$repoinfo->{'prpsearchpath'} || []};
    while (@sprp) {
      my $sprp = shift @sprp;
      my $sprojid = $sprp->{'project'};
      my $srepoid = $sprp->{'repository'};
      my $r = {};
      $r->{'recommended'} = @sprp || !@repos ? 'true' : 'false';
      $r->{'name'} = $sprojid;
      if ($nprojpack{$sprojid}) {
        $r->{'summary'} = $nprojpack{$sprojid}->{'title'};
        $r->{'description'} = $nprojpack{$sprojid}->{'description'};
      }
      my $url = BSUrlmapper::get_downloadurl("$sprojid/$srepoid");
      next unless defined $url;
      $r->{'url'} = $url;
      push @repos, $r;
    }
    $group->{'repositories'} = {'repository' => \@repos };
    my @software;
    for my $entry (@{$pat->{'rpm:requires'}->{'rpm:entry'} || []}) {
      next if $entry->{'kind'} && $entry->{'kind'} ne 'package';
      push @software, {'name' => $entry->{'name'}, 'summary' => "The $entry->{'name'} package", 'description' => "The $entry->{'name'} package."};
      fillpkgdescription($software[-1], $extrep, $repoinfo, $entry->{'name'});
    }
    for my $entry (@{$pat->{'rpm:recommends'}->{'rpm:entry'} || []}) {
      next if $entry->{'kind'} && $entry->{'kind'} ne 'package';
      push @software, {'name' => $entry->{'name'}, 'summary' => "The $entry->{'name'} package", 'description' => "The $entry->{'name'} package."};
      fillpkgdescription($software[-1], $extrep, $repoinfo, $entry->{'name'});
    }
    for my $entry (@{$pat->{'rpm:suggests'}->{'rpm:entry'} || []}) {
      next if $entry->{'kind'} && $entry->{'kind'} ne 'package';
      push @software, {'recommended' => 'false', 'name' => $entry->{'name'}, 'summary' => "The $entry->{'name'} package", 'description' => "The $entry->{'name'} package."};
      fillpkgdescription($software[-1], $extrep, $repoinfo, $entry->{'name'});
    }
    $group->{'software'} = { 'item' => \@software };
    push @{$ymp->{'group'}}, { %$group, 'distversion' => $_ } for @ympdist;
    $ymp->{'group'} ||= [ $group ];

    writexml("$extrep/.$ympname", "$extrep/$ympname", $ymp, $BSXML::ymp);

    # write database entry
    my $ympidx = {'type' => 'ymp'};
    $ympidx->{'name'} = $pat->{'name'} if defined $pat->{'name'};
    $ympidx->{'summary'} = $pat->{'summary'}->[0]->{'_content'} if $pat->{'summary'};;
    $ympidx->{'description'} = $pat->{'description'}->[0]->{'_content'} if $pat->{'description'};
    $ympidx->{'path'} = $repoinfo->{'prpsearchpath'} if $repoinfo->{'prpsearchpath'};
    $data->{'patterninfo'}->{$ympname} = $ympidx;
    db_store($patterndb, "$prp_ext/$ympname", $ympidx) if $patterndb;
  }
}

sub deletepatterns_ymp {
  my ($extrep, $projid, $repoid) = @_;

  my $prp_ext = "$projid/$repoid";
  $prp_ext =~ s/:/:\//g;
  my $patterndb = db_open('pattern');
  for my $ympname (ls($extrep)) {
    next unless $ympname =~ /\.ymp$/;
    db_store($patterndb, "$prp_ext/$ympname", undef) if $patterndb;
    unlink("$extrep/$ympname");
  }
}

##########################################################################

sub sync_to_stage {
  my ($prp, $extrep, $dbgsplit, $stageservers, $triggerstring, $isdelete, $sigs) = @_;

  my @stageservers = @{$stageservers || []};
  if (!$stageservers && $BSConfig::stageserver) {
    if (ref($BSConfig::stageserver)) {
      my @s = @{$BSConfig::stageserver};
      while (@s) {
        my ($k, $v) = splice(@s, 0, 2);
        if ($prp =~ /^$k/) {
	  $v = [ $v ] unless ref $v;
	  @stageservers = @$v;
	  last;
	}
      }
    } else {
      push @stageservers, $BSConfig::stageserver;
    }
  }

  return unless @stageservers;

  my $syncroot;
  my $extdir;

  if ($stageservers) {
    $extdir = '.';
    $syncroot = $extrep;
  } else {
    return unless $extrep =~ /\Q$extrepodir\E\/(.+)$/;
    $extdir = $1;
    $syncroot = $extrepodir;
    $triggerstring = $extdir unless $triggerstring;
  }

  # sync the parent directory for deletes
  my $extdirx = $extdir;
  $extdirx =~ s/\/[^\/]*$// if $isdelete;

  my @rsync_extra_options;
  @rsync_extra_options = split(' ', $BSConfig::rsync_extra_options) if $BSConfig::rsync_extra_options;

  for my $stageserver (@stageservers) {
    if ($stageserver =~ /^rsync(-ssl)?:\/\//) {
      my $rsynccommand = "rsync";
      if ($stageserver =~ /^rsync-ssl:/) {
        $stageserver =~ s/^rsync-ssl:/rsync:/;
        $rsynccommand = "rsync-ssl";
      }
      $stageserver .= $dbgsplit if $dbgsplit && $stageservers;
      print "    running $rsynccommand to $stageserver at ".localtime(time)."\n";
      # rsync with a timeout of 1 hour
      # sync first just the binaries without deletion of the old ones, afterwards the rest(esp. meta data) and cleanup
      qsystem('echo', "$extdirx\0", $rsynccommand, '-ar0', @rsync_extra_options, '--fuzzy', @binsufsrsync, '--include=*/', '--exclude=*', '--timeout', '7200', '--files-from=-', $syncroot, $stageserver) && die("    rsync failed at ".localtime(time).": $?\n");
      qsystem('echo', "$extdirx\0", $rsynccommand, '-ar0', @rsync_extra_options, '--delete-after', '--exclude=repocache', '--delete-excluded', '--timeout', '7200', '--files-from=-', $syncroot, $stageserver) && die("    rsync failed at ".localtime(time).": $?\n");
    }
    if ($stageserver =~ /^script:(\/.*)$/) {
      print "    running sync script $1 at ".localtime(time)."\n";
      if ($isdelete) {
        qsystem($1, $prp) && die("    sync script failed at ".localtime(time).": $?\n");
      } else {
        qsystem($1, $prp, $extdirx) && die("    sync script failed at ".localtime(time).": $?\n");
      }
    }
    if (@{$sigs || []} && $stageserver =~ /^rekor:\/\//) {
      $stageserver =~ s/^rekor:/rekor+https:/;
      $stageserver =~ s/^rekor\+//;
      my $nsigs = @{$sigs || []};
      print "    uploading $nsigs signatures to $stageserver\n";
      for my $sig (@$sigs) {
	BSRekor::upload_rekord($stageserver, $sig->{'data'}, $sig->{'pubkey'}, $sig->{'signature'}, $sig->{'format'});
      }
    }
  }

  # skip trigger unless configured
  return unless $triggerstring;
  $triggerstring .= "\0";
  my $stageserver_sync = $BSConfig::stageserver_sync;
  return unless $stageserver_sync && $stageserver_sync =~ /^rsync(-ssl)?:\/\//;

  # push done trigger sync to other mirrors
  mkdir_p($extrepodir_sync);
  my $filename = $prp;
  $filename =~ s/\//_/g;
  $filename .= $dbgsplit if $dbgsplit;
  writestr("$extrepodir_sync/.$$:$filename", "$extrepodir_sync/$filename", $triggerstring);
  my $rsynccommand = "rsync";
  if ($stageserver_sync =~ /^rsync-ssl:/) {
    $stageserver_sync =~ s/^rsync-ssl:/rsync:/;
    $rsynccommand = "rsync-ssl";
  }
  print "    running trigger rsync to $BSConfig::stageserver_sync at ".localtime(time)."\n";
  # small sync, timout 1 minute
  qsystem($rsynccommand, '-a', @rsync_extra_options, '--timeout', '120', "$extrepodir_sync/$filename", $stageserver_sync."/".$filename) && warn("    trigger rsync failed at ".localtime(time).": $?\n");
}

my $blobdir_idx;

sub create_blobdir {
  my $blobdir = "$uploaddir/publisher.$$.blobs";
  BSUtil::cleandir($blobdir);
  mkdir_p($blobdir);
  $blobdir_idx = 0;
  return $blobdir;
}

sub clean_blobdir {
  my ($blobdir) = @_;
  return unless $blobdir;
  for my $blobid (sort(ls($blobdir))) {
    unlink("$blobdir/$blobid");
    BSPublisher::Blobstore::blobstore_chk($blobid) if $blobid =~ /^_blob/;
  }
  rmdir($blobdir);
}

sub tmpfile_blobdir {
  my ($blobdir) = @_;
  my $idx = $blobdir_idx++;
  return "$blobdir/$idx";
}

sub deleterepo {
  my ($projid, $repoid, $dbgsplit) = @_;
  my $prp = "$projid/$repoid";
  print "    deleting repository\n";
  my ($extrep, $stageservers, $triggerstring) = BSUrlmapper::get_extrep_stageservers($prp);
  return unless $extrep;
  my $prp_ext = $prp;
  $prp_ext =~ s/:/:\//g;
  $extrep .= $dbgsplit if $dbgsplit;
  $prp_ext .= $dbgsplit if $dbgsplit;

  if (! -d $extrep) {
    if ($extrep =~ /\Q$extrepodir\E\/(.+)$/) {
      my $extdir = $1;
      $extdir =~ s/\/.*?$//;
      rmdir("$extrepodir/$extdir");
    }
    return if $dbgsplit;
    print "    nothing to delete...\n";
    unlink("$reporoot/$prp/:repoinfo");
    rmdir("$reporoot/$prp");
    return;
  }

  # get old repoinfo
  my $repoinfo = {};
  if (-s "$reporoot/$prp/:repoinfo") {
    $repoinfo = BSUtil::retrieve("$reporoot/$prp/:repoinfo", 1) || {};
    delete $repoinfo->{'splitdebug'} if $dbgsplit;	# do not recurse!
  }
  # delete all containers from the registories
  if ($repoinfo->{'container_repositories'}) {
    BSPublisher::Container::delete_container_repositories($extrep, $projid, $repoid, $repoinfo->{'container_repositories'});
  }
  
  # delete all binaries
  my $subdir = $repoinfo->{'subdir'} || '';
  my @archs = sort(ls($extrep));
  if ($subdir) {
    @archs = map {$_ eq $subdir ? sort(map {"$subdir/$_"} ls("$extrep/$subdir")) : $_} @archs;
  }
  my @db_deleted;
  for my $arch (@archs) {
    next if $arch =~ /^\./;
    next if $arch eq 'repodata' || $arch eq 'repocache' || $arch eq 'media.1' || $arch eq 'descr' || $arch eq 'boxes';
    my $r = "$extrep/$arch";
    next unless -d $r;
    for my $bin (ls($r)) {
      next if $bin eq 'media_info';
      my $p = "$arch/$bin";
      print "      - $p\n";
      if (-d "$r/$bin") {
        BSUtil::cleandir("$r/$bin");
        rmdir("$r/$bin") || die("rmdir $r/$bin: $!\n");
      } else {
        unlink("$r/$bin") || die("unlink $r/$bin: $!\n");
      }
      push @db_deleted, $p if $p =~ /\.(?:$binsufsre)$/;
    }
  }

  # update published database
  my $binarydb = db_open('binary');
  updatebinaryindex($binarydb, [ map {"$prp_ext/$_"} @db_deleted ], []) if $binarydb;

  my $repoinfodb = db_open('repoinfo');
  db_store($repoinfodb, $dbgsplit ? "$prp$dbgsplit" : $prp, undef) if $repoinfodb;

  if ($BSConfig::markfileorigins) {
    for my $f (sort @db_deleted) {
      my $req = {
        'uri' => "$BSConfig::markfileorigins/$prp_ext/$f",
        'request' => 'HEAD',
        'maxredirects' => 3,
        'timeout' => 10,
        'ignorestatus' => 1,
      };
      eval {
        BSRPC::rpc($req, undef, 'cmd=deleted');
      };
      print "      $f: $@" if $@;
    }
  }
  # delete ymps so they get removed from the database
  deletepatterns_ymp($extrep, $projid, $repoid);
  # delete everything else
  qsystem('rm', '-rf', $extrep);

  # delete on stage servers
  sync_to_stage($prp, $extrep, $dbgsplit, $stageservers, $triggerstring, 1);

  # also delete the split debug repo
  deleterepo($projid, $repoid, $repoinfo->{'splitdebug'}) if $repoinfo->{'splitdebug'} && !$dbgsplit;

  my $do_packtrack;
  if ($BSConfig::packtrack && (($repoinfo->{'projectkind'} || '') eq 'maintenance_release' || grep {$prp =~ /^$_/} @$BSConfig::packtrack)) {
    $do_packtrack = 1;
  }
  if (!$dbgsplit && $do_packtrack) {
    my $packtrack = {};
    print "    sending binary release tracking notification\n";
    BSNotify::notify('PACKTRACK', { project => $projid , 'repo' => $repoid }, Storable::nfreeze([ map { $packtrack->{$_} } sort keys %$packtrack ]));
  }

  # delete repoinfo
  unlink("$reporoot/$prp/:repoinfo") unless $dbgsplit;
  rmdir("$reporoot/$prp");
}

sub mapsleimage {
  my ($data, $rdir, $rbin, $p) = @_;
  my $origp = $p;
  if ($p =~ /-Build\d+/) {
    $rbin =~ s/\.asc$//;
    $rbin =~ s/\.sha256$//;
    $rbin =~ s/\.sha256.asc$//;
    $rbin =~ s/\.(?:bz2|gz|xz|zst)$//;
    $rbin =~ s/\.(spdx|cdx)\.json$//;
    $rbin =~ s/\.(install\.iso|iso|packages|raw|tbz|tgz|tar|qcow2|vdi|vhdfixed|vhdx|vmdk|vmx|vagrant\..*\.box)$//;
    $rbin =~ s/^ChangeLog\.//;
    $rbin =~ s/\.txt$//;
    # if we did not find it yet, try removing the build number
    # as the ftp trees are stored without
    $rbin =~ s/-Build\d+(\.\d+)?(\.\d+)?(\.\d+)?// unless -f "$rdir/$rbin.milestone";
    my $milestone = readstr("$rdir/$rbin.milestone", 1);
    chomp($milestone) if defined $milestone;
    $milestone ||= 'GM';
    $milestone = Build::Rpm::expandmacros($data->{'config'}, $milestone) if $milestone =~ /%/;
    $milestone =~ s/[\/\n]/_/sg;
    $p =~ s/-Build\d+(\.\d+)?(\.\d+)?(\.\d+)?/-$milestone/;
  }
  $p =~ s/-Media/-DVD/ if $data->{'media'} eq 'dvd';
  $data->{'mapped'}->{$origp} = $p if $p ne $origp;
  return $p;
}

sub depgp {
  my ($content) = @_;
  if ($content =~ /-----BEGIN PGP SIGNED MESSAGE-----\n/s) {
    $content =~ s/.*-----BEGIN PGP SIGNED MESSAGE-----//s;
    $content =~ s/.*?\n\n//s;
    $content =~ s/-----BEGIN PGP SIGNATURE-----.*//s;
    return ($content, 1);
  }
  return ($content, 0);
}

sub fixsleimage {
  my ($data, $p, $origin) = @_;
  return 0 unless $origin;
  die unless -e $origin;
  return 0 unless -f _ && -s _ < 1000000;
  my $pdir = '';
  $pdir = $1 if $p =~ /(.*\/)/;
  my $changed;
  my ($content, $hadsig) = depgp(readstr($origin));
  my $newcontent = '';
  my $mapped = $data->{'mapped'};
  for (split("\n", $content)) {
    if (/  (.*)$/ && $mapped->{"$pdir$1"}) {
      my $f = $1;
      s/  .*/  $mapped->{"$pdir$f"}/;
      s/  \Q$pdir\E/  / if $pdir ne '';
      $changed = 1;
    }
    $newcontent .= "$_\n";
  }
  return 0 unless $changed;
  my $extrep = $data->{'extrep'};
  my $bins_id = $data->{'bins_id'};
  my @s = lstat("$extrep/$p");
  if (-d _) {
    BSUtil::cleandir("$extrep/$p");
    rmdir("$extrep/$p") || die("rmdir $extrep/$p: $!\n");
    print "      ! $p\n";
  } elsif (-f _ && -s _ < 1000000) {
    my ($econtent, $ehadsig) = depgp(readstr("$extrep/$p"));
    if ($newcontent eq $econtent && $hadsig == $ehadsig) {
      $bins_id->{$p} = "$s[9]/$s[7]/$s[1]";
      return 0 unless $bins_id->{"$p.asc"};
      if (-s "$extrep/$p.asc") {
	@s = stat(_);
        $bins_id->{"$p.asc"} = "$s[9]/$s[7]/$s[1]";
        return 0;
      }
    }
    print "      ! $p\n";
  } else {
    print "      + $p\n";
  }
  unlink("$extrep/$p");
  mkdir_p(($p =~ /(.*)\//) ? "$extrep/$1" : $extrep);
  writestr("$extrep/$p", undef, $newcontent);
  if ($BSConfig::sign && ($hadsig || $bins_id->{"$p.asc"})) {
    if (!$data->{'signargs'}) {
      my $projid = $data->{'projid'};
      my ($pubkey, $signargs) = getsigndata($projid);
      my @signargs;
      push @signargs, '--project', $projid if $BSConfig::sign_project;
      push @signargs, @$signargs;
      $data->{'signargs'} = \@signargs;
    }
    my @signargs = @{$data->{'signargs'}};
    if ($hadsig) {
      qsystem($BSConfig::sign, @signargs, '-c', "$extrep/$p") && die("    sign failed: $?\n");
    }
    if ($bins_id->{"$p.asc"}) {
      if (-e "$extrep/$p.asc") {
	print "      ! $p.asc\n";
      } else {
	print "      + $p.asc\n";
      }
      unlink("$extrep/$p.asc");
      qsystem($BSConfig::sign, @signargs, '-d', "$extrep/$p") && die("    sign failed: $?\n");
      @s = stat("$extrep/$p.asc");
      die unless @s;
      $bins_id->{"$p.asc"} = "$s[9]/$s[7]/$s[1]";
    }
  }
  @s = stat("$extrep/$p");
  die unless @s;
  $bins_id->{$p} = "$s[9]/$s[7]/$s[1]";
  return 1;
}

sub linkintoblobdir {
  my ($file, $blobdirref) = @_;
  $$blobdirref ||= create_blobdir();
  my $bfile = tmpfile_blobdir($$blobdirref);
  link($file, $bfile) || die("link $file $bfile: $!\n");
  return $bfile;
}

sub readcontainermetafiles {
  my ($containerinfo, $containerinfofile, $blobdirref) = @_;
  return unless $containerinfo;
  my $prefix = $containerinfofile;
  return unless $prefix =~ s/\.containerinfo$//;
  if (-e "$prefix.slsa_provenance.json") {
    my $provenance = readstr("$prefix.slsa_provenance.json", 1);
    $containerinfo->{'slsa_provenance'} = $provenance if $provenance;
  }
  if (-e "$prefix.spdx.json") {
    $containerinfo->{'spdx_file'} = linkintoblobdir("$prefix.spdx.json", $blobdirref);
  }
  if (-e "$prefix.cdx.json") {
    $containerinfo->{'cyclonedx_file'} = linkintoblobdir("$prefix.cdx.json", $blobdirref);
  }
  $prefix =~ s/\.docker$// unless -e "$prefix.packages";
  if (-e "$prefix.packages") {
    $containerinfo->{'container_packages'} = linkintoblobdir("$prefix.packages", $blobdirref);
  }
  if (-e "$prefix.basepackages") {
    $containerinfo->{'container_basepackages'} = linkintoblobdir("$prefix.basepackages", $blobdirref);
  }
  if (-e "$prefix.report") {
    $containerinfo->{'container_report'} = linkintoblobdir("$prefix.report", $blobdirref);
  }
}

sub publish {
  my ($projid, $repoid, $dbgsplit, $dbgpacktrack, $dbgsources) = @_;
  my $prp = "$projid/$repoid";
  BSUtil::printlog("publishing $prp");
  my $starttime = time();
  my $publishid;

  # get info from source server about this project/repository
  # we specify "withsrcmd5" so that we get the patternmd5. It still
  # works with "nopackages".
  my $projpack = BSRPC::rpc("$BSConfig::srcserver/getprojpack", $BSXML::projpack, 'withrepos', 'expandedrepos', 'withsrcmd5', 'nopackages', "project=$projid", "repository=$repoid");
  if (!$projpack->{'project'}) {
    # project is gone
    deleterepo($projid, $repoid);
    return;
  }
  my $proj = $projpack->{'project'}->[0];
  die("no such project $projid\n") unless $proj && $proj->{'name'} eq $projid;
  if (!$proj->{'repository'}) {
    # repository is gone
    deleterepo($projid, $repoid);
    return;
  }
  my $repo = $proj->{'repository'}->[0];
  die("no such repository $repoid\n") unless $repo && $repo->{'name'} eq $repoid;
  # this is the already expanded path as we used 'expandedrepos' above
  my $prpsearchpath = $repo->{'path'};

  # we need the config for repotype/patterntype
  my $config = BSRPC::rpc("$BSConfig::srcserver/getconfig", undef, "project=$projid", "repository=$repoid");
  $config = Build::read_config('noarch', [ split("\n", $config) ]);

  if (!@{$config->{'repotype'} || []}) {
    # guess repotype from binarytype
    my $binarytype = $config->{'binarytype'} || '';
    my $repotype;
    $repotype = 'rpm-md' if $binarytype eq 'rpm';
    $repotype = 'debian' if $binarytype eq 'deb';
    $repotype = 'arch' if $binarytype eq 'arch';
    $repotype ||= 'rpm-md';
    $config->{'repotype'} = [ $repotype ];
  }

  my %repotype;
  for (@{$config->{'repotype'} || []}) {
    if (/^(.*?):(.*)$/) {
      $repotype{$1} = [ split(':', $2) ];
    } else {
      $repotype{$_} = [];
    }
  }
  $dbgsplit ||= '' if $repotype{'splitdebug'} && $repotype{'splitdebug'}->[0];
  my $archorder;
  $archorder = $repotype{'archorder'} if $repotype{'archorder'};

  # do we need to do package tracking?
  my $do_packtrack;
  if ($BSConfig::packtrack && (($proj->{'kind'} || '') eq 'maintenance_release' || grep {$prp =~ /^$_/} @$BSConfig::packtrack)) {
    $do_packtrack = 1;
  }

  # do we need to do source publishing?
  my $do_sourcepublish;
  if ($BSConfig::sourcepublish_sync) {
    if ($BSConfig::sourcepublish_filter) {
      $do_sourcepublish = 1 if grep {$prp =~ /^$_/} @$BSConfig::sourcepublish_filter;
    } else {
      $do_sourcepublish = 1;
    }
  }

  # is there a special subdirectory for binary packages configured?
  my $subdir = '';
  if ($repotype{'packagesubdir'} && $repotype{'packagesubdir'}->[0]) {
    $subdir = $repotype{'packagesubdir'}->[0];
    BSVerify::verify_filename($subdir);
  }
  
  my ($extrep, $stageservers, $triggerstring) = BSUrlmapper::get_extrep_stageservers($prp);
  return unless $extrep;
  my $prp_ext = $prp;
  $prp_ext =~ s/:/:\//g;
  $extrep .= $dbgsplit if $dbgsplit;
  $prp_ext .= $dbgsplit if $dbgsplit;

  # get us the lock
  local *F;
  open(F, '>', "$reporoot/$prp/.finishedlock") || die("$reporoot/$prp/.finishedlock: $!\n");
  if (!flock(F, LOCK_EX | LOCK_NB)) {
    print "    waiting for lock...\n";
    flock(F, LOCK_EX) || die("flock: $!\n");
    print "    got the lock...\n";
  }

  # we now know that $reporoot/$prp/*/:repo will not change.
  # Build repo by mixing all architectures.
  my @archs = @{$repo->{'arch'} || []};
  my %bins;
  my %bins_id;
  my $binaryorigins = {};

  my @updateinfos;
  my $updateinfos_state;

  my $modulemds = '';

  my $appdatas;
  my $appdatas_state;
  my %appdatas_seen;

  my %deltas;	# XXX remove hack
  my %deltainfos;
  my $deltainfos_state;

  my %kiwireport;	# store collected report (under the original name)
  my %kiwimedium;	# maps published name to original name
  my $kiwiindex = '';	# store collected index parts

  my %containers;

  # do iso image renaming where configured
  my $sleimagedata;
  if ($repotype{'sleimage'} || $repotype{'sleimage-media'}) {
    my $media = $repotype{'sleimage'} ? 'dvd' : $repotype{'sleimage-media'} ? 'media' : undef;
    $sleimagedata = { 'projid' => $projid, 'extrep' => $extrep, 'bins_id' => \%bins_id, 'media' => $media, 'config' => $config, 'mapped' => {} };
  }

  if ($archorder) {
    my %archorder = map {$_ => 1} @$archorder;
    my %archs = map {$_ => 1} @archs;
    # last one wins in code below
    @archs = ((grep {!$archorder{$_}} @archs), (grep {$archs{$_}} reverse(@$archorder)));
  } else {
    # just sort reverse lexicographically. This makes noarch come from x86_64, which
    # is also what the product builder does. If you really want the order of the
    # repos in the project then you can use a dummy archorder.
    @archs = sort {$b cmp $a} @archs;
  }

  # drop entire repo it source :repo's have disappeared altogether. We need to find a way
  # to specify this explicit instead checking all :repo's.
  my $found_repo;
  for my $arch (@archs) {
    my $r = "$reporoot/$prp/$arch/:repo";
    $found_repo = 1 if -e $r;
  }
  # no published binary found, but we may want an empty repo nevertheless?
  $found_repo = 1 if $config->{'publishflags:create_empty'};
  if (!defined($found_repo)) {
    deleterepo($projid, $repoid);
    return;
  }

  if ($BSConfig::publisher_compile_content_hook && $BSConfig::publisher_compile_content_hook->{$prp}) {
    my $hook = $BSConfig::publisher_compile_content_hook->{$prp};
    $hook = [ $hook ] unless ref $hook;
    print "    calling publish compile hook @$hook\n";
    qsystem(@$hook, $prp) && warn("    @$hook failed: $?\n");
    my $r = "${reporoot}.add/$prp";
    for my $rbin (sort(ls($r))) {
      my $p = "iso/$rbin";
      my @s = stat("$r/$rbin");
      $bins{$p} = "$r/$rbin";
      $bins_id{$p} = "$s[9]/$s[7]/$s[1]";
      $binaryorigins->{$p} = undef;
    }
  }

  # put containers in arch subdir?
  my $multicontainer = 0;
  $multicontainer = 1 if @archs > 1;

  my $blobdir;
  my @chksumfiles;
  for my $arch (@archs) {
    my $r = "$reporoot/$prp/$arch/:repo";
    my $repoinfo = {};
    if (-s "${r}info") {
      $repoinfo = BSUtil::retrieve("${r}info") || {};
    }
    $repoinfo->{'binaryorigins'} ||= {};

    # merge and process checksums
    if (-s "$r/.newchecksums") {
      print "    merging new checksums in $arch\n";
      my $checksums = BSUtil::retrieve("$r/.checksums", 1) || {};
      my $newchecksums = BSUtil::retrieve("$r/.newchecksums", 1) || {};
      my %knownpackids = map {$_ => 1} values(%{$repoinfo->{'binaryorigins'}});
      $checksums->{$_} = delete $newchecksums->{$_} for keys %$newchecksums;
      for my $packid (keys %$checksums) {
        delete $checksums->{$packid} unless $knownpackids{$packid};
      }
      if (%$checksums) {
        BSUtil::store("$r/.checksums.new", "$r/.checksums", $checksums);
        push @chksumfiles, "$r/.checksums";
      } else {
        unlink("$r/.checksums");
      }
      unlink("$r/.newchecksums");
    } elsif (-s "$r/.checksums") {
      push @chksumfiles, "$r/.checksums";
    }

    for my $rbin (sort(ls($r))) {
      next if $rbin eq '.newchecksums' || $rbin eq '.newchecksums.new' || $rbin eq '.checksums' || $rbin eq '.checksums.new';
      my $bin = $rbin;
      my $containerinfo;
      my $kiwimedium;

      if ($bin =~ /:updateinfo.xml$/) {
        # collect updateinfo data
        my $updateinfoxml = readstr("$r/$bin", 1) || '';
	$updateinfos_state .= Digest::MD5::md5_hex($updateinfoxml);
	my $updateinfo = readxml("$r/$bin", $BSXML::updateinfo, 1) || {};
	push @updateinfos, @{$updateinfo->{'update'} || []};
      }
      if ($bin =~ /:_modulemd.yaml$/) {
        my @s = stat("$r/$bin");
	$modulemds .= readstr("$r/$bin") if @s && $s[7] < 1024 * 1024;
	next;
      }
      if ($bin =~ /[-.]appdata.xml$/) {
        # collect application data
        my $appdataxml = readstr("$r/$bin", 1) || '';
	my $appdatamd5 = Digest::MD5::md5_hex($appdataxml);
	next if $appdatas_seen{$appdatamd5};
	$appdatas_seen{$appdatamd5} = 1;
	$appdatas_state .= $appdatamd5;
        $appdatas = merge_package_appdata($appdatas, "$arch/:repo/$bin", $appdataxml);
      }
      if ($bin =~ /^(.*\.rpm)::(.*\.drpm)$/) {
	# special drpm handling: only take it if we took the corresponding rpm
        if ($bin =~ /^(.+-[^-]+-[^-]+\.([a-zA-Z][^\/\.\-]*)\.rpm)::(.*\.drpm)$/) {
	  if ($bins{$subdir ? "$subdir/$2/$1" : "$2/$1"} eq "$r/$1") {
	    # ok, took it. also take delta
	    $bin = $3;
	    push @{$deltas{"$r/$1"}}, "$r/$rbin";
	  }
	}
      }
      if ($bin =~ /^_blob\.sha256:[0-9a-f]{64}$/) {
	# container blobid, collect
	$blobdir ||= create_blobdir();
	if (! -e "$blobdir/$bin") {
	  link("$r/$bin", "$blobdir/$bin") || die("link $r/$bin: $blobdir/$bin$!\n");
	}
      }
      $bin =~ s/^.*?:://;	# strip package name for now
      my $p;
      if ($bin =~ /^.+-[^-]+-[^-]+\.([a-zA-Z][^\/\.\-]*)\.d?rpm$/) {
	$p = "$1/$bin";
	$p = $1 eq 'src' || $1 eq 'nosrc' ? "SRPMS/$bin" : "RPMS/$bin" if $repotype{'resarchhack'};
      } elsif ($bin =~ /^.+_[^_]+_([^_\.]+)\.[ud]?deb$/) {
	$p = "$1/$bin";
      } elsif ($bin =~ /\.(?:AppImage|AppImage.zsync|snap|exe)?$/) {
	$p = "$bin";
      } elsif ($bin =~ /\.appx$/) {
	$p = "$bin";
      } elsif ($bin =~ /\.d?rpm$/) {
	# legacy format
	my $q = Build::query("$r/$rbin", 'evra' => 1);
	next unless $q;
	$p = "$q->{'arch'}/$q->{'name'}-$q->{'version'}-$q->{'release'}.$q->{'arch'}.rpm";
      } elsif ($bin =~ /\.deb$/) {
	# legacy format XXX no udeb handling
	my $q = Build::query("$r/$rbin", 'evra' => 1);
	$p = "$q->{'arch'}/$q->{'name'}_$q->{'version'}";
	$p .= "-$q->{'release'}" if defined $q->{'release'};
	$p .= "_$q->{'arch'}.deb";
      } elsif ($bin =~ /\.pkg\.tar\.(?:gz|xz|zst)(?:\.sig)?$/) {
	# special arch linux handling
	$p = "$arch/$bin";
	$p = "i686/$bin" if $arch eq 'i586';	# HACK
      } elsif ($bin =~ /\.(?:$binsufsre)$/) {
	# our default
	my $q = Build::query("$r/$rbin", 'evra' => 1);
	next unless $q && defined($q->{'arch'});
	$p = "$q->{'arch'}/$bin";
      } elsif ($bin =~ /\.slsa_provenance\.json$/) {
	next;	# we pick them up with the binaries
      } else {
	if ($bin =~ /\.iso(?:\.sha256(?:\.asc)?)?$/) {
	  $p = "iso/$bin";
	  $kiwimedium = "$arch/$1" if $bin =~ /(.+)\.iso$/;
	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
        } elsif ($bin =~ /(.*)\.(spdx|cdx)\.json$/ && -e "$r/$1.iso") {
	  $p = "iso/$bin";
    	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
	} elsif ($bin =~ /ONIE\.bin(?:\.sha256(?:\.asc)?)?$/) {
	  $p = "onie/$bin";
	  $kiwimedium = "$arch/$1" if $bin =~ /(.+)ONIE\.bin$/;
	} elsif ($bin =~ /(.*)\.raw(?:\.install)?(?:\.(?:gz|bz2|xz|zst|zstd))?(?:\.sha256(?:\.asc)?)?$/) {
	  $p = $bin;
	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
	  $kiwimedium = "$arch/$1" if !$2 && -e "$r/$1.packages";
	} elsif ($bin =~ /(.*)\.install.tar?(?:\.sha256(?:\.asc)?)?$/) {
          # kiwi has uncompressed install.tar (because internal compressed) since 17 Jul 2019
	  $p = $bin;
	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
	  $kiwimedium = "$arch/$1" if !$2 && -e "$r/$1.packages";
	} elsif ($bin =~ /(.*)\.tgz$/ && -e "$r/$1.helminfo") {
	  my @s = stat("$r/$rbin");
	  next unless @s;
	  eval { $containerinfo = BSPublisher::Helm::readhelminfo($r, "$1.helminfo") };
          if ($containerinfo) {
	    $p = $bin;
	    $containerinfo->{'arch'} = $arch;
	    $containerinfo->{'_id'} = "$s[9]/$s[7]/$s[1]";
	    $containerinfo->{'_origin'} = $repoinfo->{'binaryorigins'}->{$rbin} if defined $repoinfo->{'binaryorigins'}->{$rbin};
	    $containerinfo->{'publishfile'} = "$extrep/$p";
	    $containers{$p} = $containerinfo;
	  }
	} elsif ($bin =~ /(.*)\.tgz.prov$/ && -e "$r/$1.helminfo") {
          eval {
	    BSPublisher::Helm::readhelminfo($r, "$1.helminfo");
	    $p = $bin;
          };
	} elsif ($bin =~ /\.(manifest|efi|verity|roothash|roothash\.p7s|usrhash|usrhash\.p7s)(?:\.(?:gz|bz2|xz|zst|zstd))?(?:\.sha[0-9]*)?$/) {
	  # split dm-verity/EFI artifacts from mkosi, or image manifest file
	  $p = "$arch/$bin";
	} elsif ($bin =~ /^SHA[0-9]*SUMS(?:\.gpg)?$/) {
	  # list of hashes (possibly signed) from mkosi
	  $p = "$arch/$bin";
	} elsif ($bin =~ /^mkosi\./) {
	  # mkosi recipe file
	  $p = "$bin";
	} elsif ($bin =~ /\.containerinfo$/) {
	  # handle the case where there is a containerinfo with no tar file
	  my @s = stat("$r/$bin");
	  next unless @s;
	  eval { $containerinfo = BSRepServer::Containerinfo::readcontainerinfo($r, $bin) };
	  next unless $containerinfo;
	  next unless $containerinfo->{'tar_manifest'} && $containerinfo->{'file'} =~ /\.tar$/ && ! -e "$r/$containerinfo->{'file'}";
	  $containerinfo->{'arch'} = $arch;
	  $containerinfo->{'_id'} = "$s[9]/$s[7]/$s[1]";
	  $containerinfo->{'_origin'} = $repoinfo->{'binaryorigins'}->{$rbin} if defined $repoinfo->{'binaryorigins'}->{$rbin};
	  $p = $multicontainer ? "$arch/$containerinfo->{'file'}" : $containerinfo->{'file'};
	  $containers{$p} = $containerinfo;
	  if ($bin =~ /((.*-Build\d.*?)(?:\.docker)?)\.containerinfo$/) {
            $kiwimedium{$p} = "$arch/$2" if -e "$r/$2.packages";
	  } elsif ($bin =~ /(.*)\.containerinfo$/) {
            $kiwimedium{$p} = "$arch/$1" if -e "$r/$1.packages";
	  }
          readcontainermetafiles($containerinfo, "$r/$bin", \$blobdir);
	  next;
	} elsif ($bin =~ /((.*-Build\d.*?)(?:\.docker)?)\.(?:tbz|tgz|tar|tar\.gz|tar\.bz2|tar\.xz)(\.sha256(?:\.asc)?)?$/) {
          # kiwi image case
          $p = $bin;
	  $kiwimedium = "$arch/$2" if !$3 && -e "$r/$2.packages";
          if (-e "$r/$1.containerinfo") {
	    my @s = stat _;
	    next unless @s;
	    eval { $containerinfo = BSRepServer::Containerinfo::readcontainerinfo($r, "$1.containerinfo") };
	    warn("$@") if $@;
	    if ($containerinfo) {
	      $p = "$arch/$bin" if $multicontainer;
	      $containerinfo->{'arch'} = $arch;
	      $containerinfo->{'_id'} = "$s[9]/$s[7]/$s[1]";
	      $containerinfo->{'_origin'} = $repoinfo->{'binaryorigins'}->{$rbin} if defined $repoinfo->{'binaryorigins'}->{$rbin};
	      $containerinfo->{'publishfile'} = "$extrep/$p";
	      undef $containerinfo if $3;
	    }
            readcontainermetafiles($containerinfo, "$r/$1.containerinfo", \$blobdir) if $containerinfo;
	  } else {
	    $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
	  }
	} elsif ($bin =~ /(.*)\.tar(?:\.(?:gz|bz2|xz))?(\.sha256(?:\.asc)?)?$/) {
          # Dockerfile case
          $p = $bin;
	  $kiwimedium = "$arch/$1" if !$2 && -e "$r/$1.packages";
          if (-e "$r/$1.containerinfo") {
	    my @s = stat _;
	    next unless @s;
	    eval { $containerinfo = BSRepServer::Containerinfo::readcontainerinfo($r, "$1.containerinfo") };
	    warn("$@") if $@;
	    if ($containerinfo) {
	      $p = "$arch/$bin" if $multicontainer;
	      $containerinfo->{'arch'} = $arch;
	      $containerinfo->{'_id'} = "$s[9]/$s[7]/$s[1]";
	      $containerinfo->{'_origin'} = $repoinfo->{'binaryorigins'}->{$rbin} if defined $repoinfo->{'binaryorigins'}->{$rbin};
	      $containerinfo->{'publishfile'} = "$extrep/$p";
	      undef $containerinfo if $2;
	    }
            readcontainermetafiles($containerinfo, "$r/$1.containerinfo", \$blobdir) if $containerinfo;
	  }
	  next if $bin =~ /-(?:appstream|desktopfiles|polkitactions|mimetypes)\.tar/;
        } elsif ($bin =~ /\.(?:tgz|zip)?(?:\.sha256(?:\.asc)?)?$/) {
          # esp. for Go builds
          $p = $bin;
        } elsif ($bin =~ /\.squashfs$/) {
	  $p = $bin;	# for simpleimage builds
	} elsif ($bin =~ /\.diff\.(?:gz)(?:\.sha256(?:\.asc)?)?$/) {
	  $p = $bin;
	} elsif ($bin =~ /\.dsc(?:\.sha256(?:\.asc)?)?$/) {
	  $p = $bin;
	} elsif ($bin =~ /\.orig\.tar\.(gz|xz|bz2)\.asc$/) {
	  # Debian upstream tarball signature
	  $p = $bin;
        } elsif ($bin =~ /^(.*)\.(?:box|json|ovf|phar|qcow2|vdi|vhdfixed|vmx|vmdk|vhdx)(?:\.xz)?(\.sha256(?:\.asc)?)?$/) {
	  $p = $bin;
          $kiwimedium = "$arch/$1" if !$2 && -e "$r/$1.packages";
	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
        } elsif ($bin =~ /\.(?:cdx|spdx).json$/) {
          next unless $config->{'publishflags:withsbom'};
          $p = $bin;
        } elsif ($bin =~ /\.packages$/) {
          # FIXME2.11: to be removed
	  next unless $config->{'publishflags:withreports'};
          $p = $bin;
	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
        } elsif ($bin =~ /^(.*)\.report$/) {
	  # collect reports
	  $kiwireport{"$arch/$1"} = readxml("$r/$rbin", $BSXML::report, 1) if $do_packtrack || $do_sourcepublish;
	  next unless $config->{'publishflags:withreports'};
	  $p = $bin;
	} elsif ($bin =~ /^(.*)\.index$/) {
	  # collect virt-builder index parts
	  $kiwiindex .= readstr("$r/$bin", 1) . "\n";
	  next;
	} elsif ($bin =~ /\.flatpak$/) {
	  $p = $bin;
	} elsif ($bin =~ /^ChangeLog\./) {
	  # ChangeLog.* files, all formats
	  # Document the changes for the downloader of the images
	  $p = $bin;
	  $p = mapsleimage($sleimagedata, "$reporoot/$prp/$arch/:repo", $rbin, $p) if $sleimagedata;
	} elsif (-d "$r/$rbin") {
	  $p = "repo/$bin";
	  if ($repotype{'slepool'}) {
	    # HACK: do fancy sle repo renaming
	    my $name = $repotype{'slepool'}->[0] || 'product';
	    $p = $bin;
	    if ($name eq 'nobuildid') {
	      $p = "repo/$bin";
	      $p =~ s/-Build[\d\.]+-/-/;
	    } elsif ($bin =~ /.*-Media1?(\.license|)$/) {
	      $p = "$name$1";
	    } elsif ($bin =~ /-Media3$/) {
	      $p = "${name}_debug";
	    } elsif ($bin =~ /-Media2$/) {
	      my $rbin3 = $rbin;
	      $rbin3 =~ s/2$/3/;
	      if (-d "$r/$rbin3") {
	        $p = "${name}_source";	# 3 media available, 2 is source
	      } else {
	        $p = "${name}_debug";	# source is on media 1, 2 is debug
	      }
	    }
	    $p = $bin if $kiwimedium{"$arch/$p"};	# what???
	  }
	  $kiwimedium = "$arch/$bin";
	} else {
	  next;
	}
      }
      next unless defined $p;
      $p = "$subdir/$p" if $subdir;
      # next if $bins{$p}; # first arch wins
      my @s = stat("$reporoot/$prp/$arch/:repo/$rbin");
      next unless @s;
      if ($bins{$p}) {
	if (!$archorder) {
          # keep old file (FIXME: should do this different)
          my @s2 = stat("$extrep/$p");
          next if !@s2 || "$s[9]/$s[7]/$s[1]" ne "$s2[9]/$s2[7]/$s2[1]";
	}
        # replace already taken binary. kill taken deltas again
        for my $d (@{$deltas{"$r/$rbin"} || []}) {
	  for my $dp (grep {$bins{$_} eq $d} keys %bins) {
	    delete $bins{$dp};
	    delete $bins_id{$dp};
	    delete $binaryorigins->{$dp};
	    delete $deltainfos{$dp};
	  }
        }
      }

      $bins{$p} = "$r/$rbin";
      $bins_id{$p} = "$s[9]/$s[7]/$s[1]";
      $binaryorigins->{$p} = $repoinfo->{'binaryorigins'}->{$rbin};
      $kiwimedium{$p} = $kiwimedium if $kiwimedium;
      $containers{$p} = $containerinfo if $containerinfo;

      if ($rbin =~ /^(.*)\.drpm$/) {
	# we took a delta rpm. collect desq if possible
	my $dseq = "$r/$1.dseq";
	if (-s $dseq) {
	  my %dseq;
	  for (split("\n", readstr($dseq, 1) || '')) {
	    $dseq{$1} = $2 if /^(.*?): (.*)$/s;
	  }
	  my @needed = qw{Name Epoch Version Release Arch OldName OldEpoch OldVersion OldRelease OldArch Seq};
	  if (!grep {!exists($dseq{$_})} @needed) {
	    # got all required fields. convert to correct data
	    my $dinfo = {'name' => $dseq{'Name'}, 'epoch' => $dseq{'Epoch'} || 0, 'version' => $dseq{'Version'}, 'release' => $dseq{'Release'}, 'arch' => $dseq{'Arch'}};
	    $dinfo->{'delta'} = [ {'oldepoch' => $dseq{'OldEpoch'} || 0, 'oldversion' => $dseq{'OldVersion'}, 'oldrelease' => $dseq{'OldRelease'}, 'filename' => $p, 'sequence' => $dseq{'Seq'}} ];
	    $deltainfos{$p} = $dinfo;
	  }
	}
      }

      if ($rbin =~ /(.*)\.(?:$binsufsre)$/) {
	# we took a binary. also take provenance if we have it
	my $rprovenance = "$1.slsa_provenance.json";
	my $provenance = $p;
	if ($provenance =~ s/\.(?:$binsufsre)$/.slsa_provenance.json/) {
	  if (-e "$r/$rprovenance") {
	    my @s = stat(_);
	    $bins{$provenance} = "$r/$rprovenance";
	    $bins_id{$provenance} = "$s[9]/$s[7]/$s[1]";
	    $binaryorigins->{$provenance} = $repoinfo->{'binaryorigins'}->{$rbin};
	    $kiwimedium{$provenance} = $kiwimedium if $kiwimedium;
	    $containers{$provenance} = $containerinfo if $containerinfo;
	  } else {
	    delete $bins{$provenance};
	    delete $bins_id{$provenance};
	    delete $binaryorigins->{$provenance};
	    delete $kiwimedium{$provenance};
	    delete $containers{$provenance};
	  }
	}
      }

    }
  }

  # calculate deltainfos_state
  if (%deltainfos) {
    $deltainfos_state = '';
    for my $p (sort keys %deltainfos) {
      my @s = stat($bins{$p});
      my $id = "$s[9]/$s[7]/$s[1]";
      if ($bins{$p} =~ /^(.*)\.drpm$/) {
        @s = stat("$1.dseq");
        $id .= "/$s[9]/$s[7]/$s[1]";
      }
      $deltainfos_state .= Digest::MD5::md5_hex($id);
    }
  }

  # do debug filtering if requested
  if (defined($dbgsplit)) {
    if ($dbgsplit) {
      for my $p (keys %bins) {
        next if $p =~ /-debug(?:info|source)-.*rpm$/;
        delete $bins{$p};
        delete $deltainfos{$p};
      }
    } else {
      for my $p (keys %bins) {
        next unless $p =~ /-debug(?:info|source)-.*rpm$/;
        delete $bins{$p};
        delete $deltainfos{$p};
      }
    }
  }

  my $changed = 0;
  my @changed;  	# all changed files for hooks.

  # decide what to do with containers
  my %containerextras;
  my @containerregistries;
  @containerregistries = BSPublisher::Container::registries_for_prp($projid, $repoid) if %containers;
  if (@containerregistries) {
    # replace all containers with readme files pointing to the registries
    for my $p (sort keys %containers) {
      my $pp = $p =~ /\/(.*)$/ ? $1 : $p;
      $containerextras{"$pp.registry.txt"} = 1;
      my $containerinfo = $containers{$p};
      if ($containerinfo->{'publishfile'}) {
	# link container into the blobdir
	$blobdir ||= create_blobdir();
	my $tmpfile = "$blobdir/$containerinfo->{'arch'}:$pp";
	link($bins{$p}, $tmpfile) || die("link $bins{$p} $tmpfile: $!\n");
	$containerinfo->{'publishfile'} = $tmpfile;
      }
      next if ($containerinfo->{'type'} || '') eq 'helm';	# keep helm charts
      delete $bins{$p};
      delete $bins{"$p.sha256"};
      delete $bins{"$p.sha256.asc"};
      delete $binaryorigins->{$p};
      delete $binaryorigins->{"$p.sha256"};
      delete $binaryorigins->{"$p.sha256.asc"};
    }
  } else {
    # reconstruct missing virtual containers
    for my $p (sort keys %containers) {
      my $pp = $p =~ /\/(.*)$/ ? $1 : $p;
      $containerextras{$pp} = 1 if $multicontainer && $p ne $pp;
      my $containerinfo = $containers{$p};
      next if $containerinfo->{'publishfile'};
      next unless $containerinfo->{'tar_sha256sum'};
      my $fd;
      if (open($fd, '<', "$extrep/$p")) {
	my @s = stat($fd);
	# can we re-use the old container?
	my $ctx = Digest->new("SHA-256");
        $ctx->addfile($fd);
	my $sha256sum = $ctx->hexdigest();
	close($fd);
	if ($sha256sum eq $containerinfo->{'tar_sha256sum'}) {
	  $bins{$p} = undef;
	  $bins_id{$p} = "$s[9]/$s[7]/$s[1]";
	  $binaryorigins->{$p} = $containerinfo->{'_origin'};
	  next;
	}
        print "      ! $p\n";
      } else {
        print "      + $p\n";
      }
      $containerinfo->{'blobdir'} = $blobdir if $blobdir;
      mkdir_p($extrep) unless -d $extrep;
      BSPublisher::Container::reconstruct_container($containerinfo, "$extrep/$p");
      my @s = stat("$extrep/$p");
      $bins{$p} = undef;
      $bins_id{$p} = "$s[9]/$s[7]/$s[1]";
      $binaryorigins->{$p} = $containerinfo->{'_origin'};
      push @changed, "$extrep/$p";
      $changed = 1;
    }
  }

  # fix sha256 checksum files of mapped iso files
  if ($sleimagedata && %{$sleimagedata->{'mapped'}}) {
    for my $p (grep {/\.sha256$/} sort keys %bins) {
      $changed = 1 if fixsleimage($sleimagedata, $p, $bins{$p});
    }
  }

  # now update external repository
  my @db_deleted;  	# for published db update
  my @db_changed;	# for published db update

  my %bins_done;
  @archs = sort(ls($extrep));
  if ($subdir) {
    @archs = map {$_ eq $subdir ? sort(map {"$subdir/$_"} ls("$extrep/$subdir")) : $_} @archs;
  }
  for my $arch (@archs) {
    next if $arch =~ /^\./;
    next if $arch eq 'repodata' || $arch eq 'repocache' || $arch eq 'media.1' || $arch eq 'descr' || $arch eq 'boxes';
    next if $arch =~ /\.repo$/;
    next if $arch eq 'Packages' || $arch eq 'Packages.gz' || $arch eq 'Sources' || $arch eq 'Sources.gz' || $arch eq 'Release' || $arch eq 'Release.gz' || $arch eq 'Release.key';
    next if $containerextras{$arch};		# keep the old readme/symlink
    my $r = "$extrep/$arch";
    if (-f $r) {
      $r = $extrep;
      my $bin = $arch;
      my $p = $arch;
      my @s = lstat("$r/$bin");
      if (!exists($bins{$p})) {
	print "      - $p\n";
        unlink("$r/$bin") || die("unlink $r/$bin: $!\n");
        push @db_deleted, $p if $p =~ /\.(?:$binsufsre)$/;
        $changed = 1;
	next;
      }
      if ("$s[9]/$s[7]/$s[1]" ne $bins_id{$p}) {
        unlink("$r/$bin") || die("unlink $r/$bin: $!\n");
        link($bins{$p}, "$r/$bin") || die("link $bins{$p} $r/$bin: $!\n");
	push @db_changed, $p if $p =~ /\.(?:$binsufsre)$/;
	push @changed, $p;
        $changed = 1;
      }
      $bins_done{$p} = 1;
      next;
    }
    if ($bins{$arch}) {
      my @s = lstat($r);
      die("$r: $!\n") unless @s;
      next if "$s[9]/$s[7]/$s[1]" eq $bins_id{$arch};
      if (-d _) {
	if (! -l $bins{$arch} && -d _) {
	  my $info1 = BSUtil::treeinfo($bins{$arch});
	  my $info2 = BSUtil::treeinfo($r);
	  if (join(',', @$info1) eq join(',', @$info2)) {
	    $bins_done{$arch} = 1;
	    next;
	  }
	  print "      ! $arch\n";
	  BSUtil::cleandir($r);
	  rmdir($r) || die("rmdir $r: $!\n");
	}
	if (! -l $bins{$arch} && -d _) {
	  BSUtil::linktree($bins{$arch}, $r);
	} else {
	  link($bins{$arch}, $r) || die("link $bins{$arch} $r: $!\n");
	}
	push @db_changed, $arch if $arch =~ /\.(?:$binsufsre)$/;
	push @changed, $arch;
	$changed = 1;
	$bins_done{$arch} = 1;
	next;
      }
    }
    next unless -d $r;
    for my $bin (sort(ls($r))) {
      my $p = "$arch/$bin";
      my @s = lstat("$r/$bin");
      die("$r/$bin: $!\n") unless @s;
      if (!exists($bins{$p})) {
	print "      - $p\n";
	if (-d _) {
	  BSUtil::cleandir("$r/$bin");
	  rmdir("$r/$bin") || die("rmdir $r/$bin: $!\n");
	} else {
	  unlink("$r/$bin") || die("unlink $r/$bin: $!\n");
	}
	push @db_deleted, $p if $p =~ /\.(?:$binsufsre)$/;
        $changed = 1;
	next;
      }
      if ("$s[9]/$s[7]/$s[1]" ne $bins_id{$p}) {
        # changed, link over
	if (-d _) {
	  if (! -l $bins{$p} && -d _) {
	    # both are directories, compare info
	    # should MIX instead?
	    my $info1 = BSUtil::treeinfo($bins{$p});
	    my $info2 = BSUtil::treeinfo("$r/$bin");
	    if (join(',', @$info1) eq join(',', @$info2)) {
	      $bins_done{$p} = 1;
	      next;
	    }
	  }
	  print "      ! $p\n";
	  BSUtil::cleandir("$r/$bin");
	  rmdir("$r/$bin") || die("rmdir $r/$bin: $!\n");
	} else {
	  print "      ! $p\n";
	  unlink("$r/$bin") || die("unlink $r/$bin: $!\n");
	}
	if (! -l $bins{$p} && -d _) {
	  BSUtil::linktree($bins{$p}, "$r/$bin");
	} else {
          link($bins{$p}, "$r/$bin") || die("link $bins{$p} $r/$bin: $!\n");
	}
	push @db_changed, $p if $p =~ /\.(?:$binsufsre)$/;
	push @changed, $p;
        $changed = 1;
      }
      $bins_done{$p} = 1;
    }
  }
  for my $p (sort keys %bins) {
    next if $bins_done{$p};
    # a new one
    my ($arch, $bin);
    if ($p =~ /^(.*)\/([^\/]*)$/s) {
      ($arch, $bin) = ($1, $2);
    } else {
      ($arch, $bin) = ('.', $p);
    }
    my $r = "$extrep/$arch";
    mkdir_p($r) unless -d $r;
    print "      + $p\n";
    if (! -l $bins{$p} && -d _) {
      BSUtil::linktree($bins{$p}, "$r/$bin");
    } else {
      link($bins{$p}, "$r/$bin") || die("link $bins{$p} $r/$bin: $!\n");
    }
    push @db_changed, $p if $p =~ /\.(?:$binsufsre)$/;
    push @changed, $p;
    $changed = 1;
  }

  # Write the kiwi index file if we got it
  if ($kiwiindex) {
    my $oldkiwiindex = readstr("$extrep/index", 1) || '';
    if ($oldkiwiindex ne $kiwiindex) {
      writestr("$extrep/index", undef, $kiwiindex);
      push @changed, "$extrep/index";
      $changed = 1;
    }
  }

  close F;     # release repository lock

  # add state for external container registries
  my $container_state = '';
  if (%containers) {
    $container_state = BSPublisher::Container::calculate_container_state($projid, $repoid, \%containers, $multicontainer);
  }

  my $modulemds_state = '';
  $modulemds_state = Digest::MD5::md5_hex($modulemds) if $modulemds;

  my $title = $proj->{'title'} || $projid;
  $title .= " ($repoid)";
  $title =~ s/\n/ /sg;

  my $state;
  $state = $proj->{'patternmd5'} || '';
  $state .= "\0".join(',', @{$config->{'repotype'} || []}) if %bins;
  $state .= "\0".($proj->{'title'} || '') if %bins;
  $state .= "\0".join(',', @{$config->{'patterntype'} || []}) if $proj->{'patternmd5'};
  $state .= "\0".join('/', map {"$_->{'project'}/$_->{'repository'}"} @{$prpsearchpath || []}) if $proj->{'patternmd5'};
  $state .= "\0".$updateinfos_state if $updateinfos_state;
  $state .= "\0".$appdatas_state if $appdatas_state;
  $state .= "\0".$deltainfos_state if $deltainfos_state;
  $state .= "\0".$container_state if $container_state;
  $state .= "\0".$modulemds_state if $modulemds_state;
  $state = Digest::MD5::md5_hex($state) if $state ne '';

  # get us the old repoinfo, so we can compare the state
  my $oldrepoinfo = {};
  my $packtrackcache;
  if (-s "$reporoot/$prp/:repoinfo") {
    $oldrepoinfo = BSUtil::retrieve("$reporoot/$prp/:repoinfo");
    $packtrackcache = $oldrepoinfo->{'trackercache'} if $oldrepoinfo->{'trackercache'} && ($oldrepoinfo->{'trackercacheversion'} || '') eq $trackercacheversion;
    $publishid = $oldrepoinfo->{'publishid'} + 1 if $oldrepoinfo->{'publishid'};
  }
  $publishid ||= $starttime;

  if (($oldrepoinfo->{'state'} || '') ne $state) {
    $changed = 1;
  }

  if (($oldrepoinfo->{'splitdebug'} || '') ne (($repotype{'splitdebug'} || [])->[0] || '')) {
    deleterepo($projid, $repoid, $oldrepoinfo->{'splitdebug'}) if $oldrepoinfo->{'splitdebug'};
    $changed = 1;
  }

  if (!$changed && !$dbgsplit) {
    print "    nothing changed\n";
    clean_blobdir($blobdir);
    return 'unchanged';
  }

  mkdir_p($extrep) unless -d $extrep;

  # get sign data
  my ($pubkey, $signargs) = getsigndata($projid);

  # get all patterns
  my $patterns = [];
  if ($proj->{'patternmd5'}) {
    $patterns = getpatterns($projid);
  }

  # upload all containers to configured registries
  my $container_repositories;
  if (%containers || $oldrepoinfo->{'container_repositories'}) {
    my $data = {
      'pubkey' => $pubkey,
      'signargs' => $signargs,
      'config' => $config,
      'publishid' => $publishid,
      'multiarch' => $multicontainer,
    };
    $data->{'notify'} = sub { BSNotify::notify('CONTAINER_PUBLISHED', { project => $projid , 'repo' => $repoid, 'buildid' => $publishid, 'container' => "$_[0]"}) };
    if ($blobdir) {
      $_->{'blobdir'} = $blobdir for values %containers;
    }
    eval {
      $container_repositories = BSPublisher::Container::upload_all_containers($extrep, $projid, $repoid, \%containers, $data, $oldrepoinfo->{'container_repositories'} || {});
    };
    if ($@) {
      clean_blobdir($blobdir);
      unlink("$uploaddir/publisher.$$") if @$signargs && $signargs->[0] eq '-P';
      die($@);
    }
  }
  clean_blobdir($blobdir);
  undef $blobdir;

  # collect packtrack and source data (if needed)
  my $packtrack;
  my $sources;
  if ($do_packtrack || $do_sourcepublish) {
    $packtrack = $dbgpacktrack || {};
    $sources = $dbgsources || {};
    my %cache = @{$packtrackcache || []};
    for my $bin (sort keys %{ { %bins, %containers } }) {
      my $reportfile;
      if ($bin =~ /\.(?:$binsufsre)$/) {
	my $res;
	my @s = stat("$extrep/$bin");
	next unless @s;
	my $c = $cache{"$bin/$s[9]/$s[7]/$s[1]"};
	if ($c) {
	  my @d = qw{arch name epoch version release disturl buildtime cpeid};
	  $res = {};
	  for (@$c) {
	    my $dd = shift @d;
	    $res->{$dd} = $_ if defined $_;
	  }
	} else {
	  eval {
            $res = Build::query("$extrep/$bin", 'evra' => 1, 'buildtime' => 1, 'disturl' => 1);
	  };
	  next unless $res;
          # catch provided product cpeid strings
	  my ($cpeid) = grep {/^product-cpeid\(\)/} @{$res->{'provides'} || []};
	  if ($cpeid) {
	    $cpeid =~ s/.* = //;
	    $res->{'cpeid'} = BSHTTP::urldecode($cpeid);
	  }
	}
	my $pt = { 'project' => $projid, 'repository' => $repoid };
	$pt->{'arch'} = $1 if $bins{$bin} =~ /^\Q$reporoot\E\/\Q$prp\E\/([^\/]+)\//;
	$pt->{'package'} = $binaryorigins->{$bin} if $binaryorigins->{$bin};
	for (qw{name epoch version release disturl buildtime}) {
	  $pt->{$_} = $res->{$_} if defined $res->{$_};
	}
	$pt->{'binaryarch'} = $res->{'arch'} if defined $res->{'arch'};
	$pt->{'binaryid'} = $res->{'hdrmd5'} if defined $res->{'hdrmd5'};
	$pt->{'id'} = "$bin/$s[9]/$s[7]/$s[1]";
	$pt->{'cpeid'} = $res->{'cpeid'} if $res->{'cpeid'};
	$packtrack->{$bin} = $pt;
	if ($pt->{'arch'}) {
	  $reportfile = $bin;
	  $reportfile =~ s/.*\///;    # basename
	  $reportfile = "$pt->{'arch'}/$reportfile";
	}
      }

      # track containers
      if ($containers{$bin}) {
	my $containerinfo = $containers{$bin};
	my $medium = $bin;
	$medium =~ s/.*\///;	# basename
	my $lnk = BSRepServer::Containerinfo::containerinfo2nevra($containerinfo);
	my $pt = { 'project' => $projid, 'repository' => $repoid };
	for (qw{name epoch version release}) {
	  $pt->{$_} = $lnk->{$_} if defined $lnk->{$_};
	}
	$pt->{'binaryarch'} = $lnk->{'arch'} || 'noarch';
	$pt->{'package'} = $containerinfo->{'_origin'} if $containerinfo->{'_origin'};
	for (qw{arch disturl buildtime}) {
	  $pt->{$_} = $containerinfo->{$_} if defined $containerinfo->{$_};
	}
        $pt->{'binaryid'} = $containerinfo->{'imageid'} if $containerinfo->{'imageid'};
	$pt->{'ismedium'} = $medium;
	$packtrack->{$bin} = $pt;
      }

      # track images and kiwi products
      if (!$containers{$bin} && $kiwimedium{$bin} && $kiwireport{$kiwimedium{$bin}}) {
	my $medium = $bin;
	$medium =~ s/.*\///;	# basename
	my $report = $kiwireport{$kiwimedium{$bin}};
	my $pt = { 'project' => $projid, 'repository' => $repoid };
	$pt->{'arch'} = (split('/', $kiwimedium{$bin}, 2))[0];
	$pt->{'package'} = $binaryorigins->{$bin} if $binaryorigins->{$bin};
	$pt->{'name'} = $medium;	# hmm...
	$pt->{'version'} = defined($report->{'version'}) ? $report->{'version'} : '0';
	$pt->{'release'} = defined($report->{'release'}) ? $report->{'release'} : '0';
	$pt->{'binaryarch'} = $report->{'binaryarch'} || $pt->{'arch'} || 'noarch';
	$pt->{'buildtime'} = $report->{'buildtime'} if $report->{'buildtime'};
	$pt->{'cpeid'} = BSHTTP::urldecode($report->{'cpeid'}) if $report->{'cpeid'};
	# need a way to get disturl, buildtime and binaryid...
	$pt->{'disturl'} = $report->{'disturl'} if $report->{'disturl'};
	$pt->{'ismedium'} = $medium;
	$packtrack->{$bin} = $pt;
      }

      # add a published source entry
      my $sourceid;
      if ($do_sourcepublish) {
        my $pt = $packtrack->{$bin};
        if ($pt->{'disturl'} && $pt->{'disturl'} =~ /^obs:\/\/[^\/]*\/([^\/]*)\/[^\/]*\/([^-]*)-(.*)$/) {
          $sourceid = "$1/$3/$2";
          $sources->{$sourceid} = [];
        }
      }

      # check if there is a report associated with this binary
      # this adds the content to the referenced binary
      $reportfile = $kiwimedium{$bin} if $kiwimedium{$bin};
      if ($reportfile && $kiwireport{$reportfile}) {
	my $medium = $bin;
	$medium =~ s/.*\///;	# basename
	my $report = $kiwireport{$reportfile};
	# make sure ismedium is set in the binary
	$packtrack->{$bin}->{'ismedium'} = $medium if $packtrack->{$bin};
	for my $kb (@{$report->{'binary'} || []}) {
	  my $pt = { %$kb };
	  delete $pt->{'_content'};
	  $pt->{'medium'} = $medium;
	  $pt->{'binaryarch'} ||= $pt->{'arch'} || 'noarch';
	  my $fn = '';
	  $fn .= "/".(defined($pt->{$_}) ? $pt->{$_} : '') for qw{binaryarch name epoch version release};
	  $packtrack->{"$medium$fn"} = $pt;
	  if ($sourceid && $pt->{'disturl'} && $pt->{disturl} =~ /^obs:\/\/[^\/]*\/([^\/]*)\/[^\/]*\/([^-]*)-(.*)$/) {
	    push @{$sources->{$sourceid}}, "$1/$3/$2";
	  }
	}
      }
    }
    # argh, find patchinforef and put it in update
    if (@updateinfos) {
      for my $up (@updateinfos) {
	for my $cl (@{($up->{'pkglist'} || {})->{'collection'} || []}) {
	  for my $pkg (@{$cl->{'package'} || []}) {
	    my $pn = ($subdir ? "$subdir/" : '') . "$pkg->{'arch'}/$pkg->{'filename'}";
	    if ($packtrack->{$pn}) {
	      $packtrack->{$pn}->{'patchinforef'} = $up->{'patchinforef'} if $up->{'patchinforef'};
	      $packtrack->{$pn}->{'updateinfoid'} = $up->{'id'};
	      $packtrack->{$pn}->{'updateinfoversion'} = $up->{'version'};
	      # XXX: do this in hook?
	      my $supportstatus = $pkg->{'supportstatus'};
	      # workaround for broken code11 imports
	      if ($supportstatus) {
	        $supportstatus =~ s/^support_//;
	        $packtrack->{$pn}->{'supportstatus'} = $supportstatus;
	      }
	    }
	  }
	}
      }
    }
  }
  undef $packtrackcache;

  # create and store the new repoinfo
  my $repoinfo = {
    'prpsearchpath' => $prpsearchpath,
    'binaryorigins' => $binaryorigins,
    'title' => $title,
    'state' => $state,
    'repotype' => \%repotype,
  };
  $repoinfo->{'projectkind'} = $proj->{'kind'} if $proj->{'kind'};
  $repoinfo->{'arch'} = $repo->{'arch'} if $repo->{'arch'};
  $repoinfo->{'splitdebug'} = $repotype{'splitdebug'}->[0] if defined $dbgsplit;
  $repoinfo->{'subdir'} = $subdir if $subdir;
  $repoinfo->{'base'} = $repo->{'base'} if $repo->{'base'};
  $repoinfo->{'container_repositories'} = $container_repositories if %{$container_repositories || {}};
  $repoinfo->{'publishid'} = $publishid;
  $repoinfo->{'code'} = 'building';
  $repoinfo->{'starttime'} = $starttime;

  # store repoinfo on disk
  if (!$dbgsplit) {
    if ($state ne '') {
      BSUtil::store("$reporoot/$prp/.:repoinfo", "$reporoot/$prp/:repoinfo", $repoinfo);
    } else {
      unlink("$reporoot/$prp/:repoinfo");
    }
  }

  # do debug filtering if requested
  if (defined($dbgsplit)) {
    for (keys %$binaryorigins) {
      delete $binaryorigins->{$_} unless $bins{$_};
    }
    if (@updateinfos) {
      my $dbgsplittype = ($repotype{'splitdebug'}->[1]) || 'mainupdateinfo';
      @updateinfos = () if $dbgsplit;
      if ($dbgsplittype ne 'mainupdateinfo' && !$dbgsplit) {
	for my $up (@updateinfos) {
	  for my $cl (@{($up->{'pkglist'} || {})->{'collection'} || []}) {
	    next unless $cl->{'package'};
	    my $haveremovedpkg;
	    for my $pkg (@{$cl->{'package'}}) {
	      next unless $pkg->{'filename'} =~ /-debug(?:info|source)-.*rpm$/;
	      $pkg = undef;
	      $haveremovedpkg = 1;
	    }
	    next unless $haveremovedpkg;
	    $cl->{'package'} = [ grep {defined($_)} @{$cl->{'package'}} ];
	  }
	}
      }
    }
  }

  # store repoinfo in published database
  my $repoinfodb = db_open('repoinfo');
  db_store($repoinfodb, $dbgsplit ? "$prp$dbgsplit" : $prp, $state ne '' ? $repoinfo : undef) if $repoinfodb;

  # put in published database
  my $binarydb = db_open('binary');
  updatebinaryindex($binarydb, [ map {"$prp_ext/$_"} @db_deleted ], [ map {"$prp_ext/$_"} @db_changed ]) if $binarydb;

  # mark file origins so we can gather per package statistics
  if ($BSConfig::markfileorigins) {
    print "    marking file origins\n";
    for my $f (sort @db_changed) {
      my $origin = $binaryorigins->{$f};
      $origin = "?" unless defined $origin;
      my $req = {
        'uri' => "$BSConfig::markfileorigins/$prp_ext/$f",
        'request' => 'HEAD',
        'maxredirects' => 3,
        'timeout' => 10,
        'ignorestatus' => 1,
      };
      eval {
        BSRPC::rpc($req, undef, 'cmd=setpackage', "package=$origin");
      };
      print "      $f: $@" if $@;
    }
    for my $f (sort @db_deleted) {
      my $req = {
        'uri' => "$BSConfig::markfileorigins/$prp_ext/$f",
        'request' => 'HEAD',
        'maxredirects' => 3,
        'timeout' => 10,
        'ignorestatus' => 1,
      };
      eval {
        BSRPC::rpc($req, undef, 'cmd=deleted');
      };
      print "      $f: $@" if $@;
    }
  }

  # create repositories and patterns
  my %patterntype;
  for (@{$config->{'patterntype'} || []}) {
    if (/^(.*?):(.*)$/) {
      $patterntype{$1} = [ split(':', $2) ];
    } else {
      $patterntype{$_} = [];
    }
  }
  if ($repotype{'rpm-md-legacy'}) {
    $repotype{'rpm-md'} = $repotype{'rpm-md-legacy'};
    unshift @{$repotype{'rpm-md'}}, 'legacy';
    delete $repotype{'rpm-md-legacy'};
  }
  if ($BSConfig::publishprogram && $BSConfig::publishprogram->{$prp}) {
    local *PPLOCK;
    open(PPLOCK, '>', "$reporoot/$prp/.pplock") || die("$reporoot/$prp/.pplock: $!\n");
    flock(PPLOCK, LOCK_EX) || die("flock: $!\n");
    if (xfork()) {
      close PPLOCK;
      return;
    }
    if (system($BSConfig::publishprogram->{$prp}, $prp, $extrep)) {
      die("      $BSConfig::publishprogram->{$prp} failed: $?\n");
    }
    goto publishprog_done;
  }

  my $xrepoid = $repoid;
  $xrepoid .= $dbgsplit if $dbgsplit;

  my @repotags = @{$repotype{'repotag'} || []};
  # de-escape (mostly done for ':'
  s/%([a-fA-F0-9]{2})/chr(hex($1))/ge for @repotags;
  if (grep {$_ eq '-obsrepository'} @repotags) {
    @repotags = grep {$_ ne '-obsrepository'} @repotags;
  } else {
    my $obsname = $BSConfig::obsname || 'build.opensuse.org';
    push @repotags, "obsrepository://$obsname/$projid/$repoid";
  }
  push @repotags, "obsbuildid:$repoinfo->{'publishid'}" if $repoinfo->{'publishid'};

  my $data = {
    'subdir' => $subdir,
    'pubkey' => $pubkey,
    'signargs' => $signargs,
    'repoinfo' => $repoinfo,
    'updateinfos' => \@updateinfos,
    'deltainfos' => \%deltainfos,
    'appdatas' => $appdatas,
    'patterns' => $patterns,
    'dbgsplit' => $dbgsplit,
    'packtrack' => $packtrack,
    'repotags' => \@repotags,
    'chksumfiles' => \@chksumfiles,
    'config' => $config,
    'patterninfo' => {},
    'modulemds' => $modulemds,
    'sigs' => [],
    'publishid' => $publishid,
  };

  if ($repotype{'rpm-md'}) {
    createrepo_rpmmd($extrep, $projid, $xrepoid, $data, $repotype{'rpm-md'});
  } else {
    deleterepo_rpmmd($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'suse'}) {
    createrepo_susetags($extrep, $projid, $xrepoid, $data, $repotype{'suse'});
  } else {
    deleterepo_susetags($extrep, $projid, $xrepoid, $data);
  }
  # Mandriva format:
  if ($repotype{'hdlist2'}) {
    createrepo_hdlist2($extrep, $projid, $xrepoid, $data, $repotype{'hdlist2'});
  } else {
    deleterepo_hdlist2($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'debian'}) {
    createrepo_debian($extrep, $projid, $xrepoid, $data, $repotype{'debian'});
  } else {
    deleterepo_debian($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'arch'}) {
    createrepo_arch($extrep, $projid, $xrepoid, $data, $repotype{'arch'});
  } else {
    deleterepo_arch($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'vagrant'}) {
    createrepo_vagrant($extrep, $projid, $xrepoid, $data, $repotype{'vagrant'});
  } else {
    deleterepo_vagrant($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'staticlinks'}) {
    createrepo_staticlinks($extrep, $projid, $xrepoid, $data, $repotype{'staticlinks'});
  } else {
    deleterepo_staticlinks($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'zyppservice'}) {
    createrepo_zyppservice($extrep, $projid, $xrepoid, $data, $repotype{'zyppservice'});
  } else {
    deleterepo_zyppservice($extrep, $projid, $xrepoid, $data);
  }
  if ($repotype{'helm'}) {
    createrepo_helm($extrep, $projid, $xrepoid, $data, $repotype{'helm'}, \%containers);
  }
  if ($repotype{'checksumsfile'}) {
    createrepo_checksumsfile($extrep, $projid, $xrepoid, $data, $repotype{'checksumsfile'});
  }

  if ($patterntype{'ymp'}) {
    createpatterns_ymp($extrep, $projid, $xrepoid, $data, $patterntype{'ymp'});
  } else {
    deletepatterns_ymp($extrep, $projid, $xrepoid, $data);
  }
  if ($patterntype{'rpm-md'}) {
    createpatterns_rpmmd($extrep, $projid, $xrepoid, $data, $patterntype{'rpm-md'});
  } else {
    deletepatterns_rpmmd($extrep, $projid, $xrepoid, $data);
  }
  if ($patterntype{'comps'}) {
    createpatterns_comps($extrep, $projid, $xrepoid, $data, $patterntype{'comps'});
  } else {
    deletepatterns_comps($extrep, $projid, $xrepoid, $data);
  }

  # virt-builder repository
  if (-e "$extrep/index") {
    createrepo_virtbuilder($extrep, $projid, $xrepoid, $data);
  }

  my $patterninfodb = db_open('patterninfo');
  db_store($patterninfodb, $dbgsplit ? "$prp$dbgsplit" : $prp, %{$data->{'patterninfo'}} ? $data->{'patterninfo'} : undef) if $patterninfodb;

publishprog_done:
  unlink("$uploaddir/publisher.$$") if @$signargs && $signargs->[0] eq '-P';

  # post process step: create directory listing for poor YaST
  if ($repotype{'suse'}) {
    unlink("$extrep/directory.yast");
    my @d = sort(ls($extrep));
    for (@d) {
      $_ .= '/' if -d "$extrep/$_";
      $_ .= "\n";
    }
    writestr("$extrep/.directory.yast", "$extrep/directory.yast", join('', @d));
  }

  # push the repo to the stage servers
  sync_to_stage($prp, $extrep, $dbgsplit, $stageservers, $triggerstring, 0, $data->{'sigs'});

  # support for regex usage in $BSConfig::unpublishedhook
  my $unpublish_prp = $prp;
  if ($BSConfig::unpublishedhook_use_regex || $BSConfig::unpublishedhook_use_regex) {
    for my $key (sort {$b cmp $a} keys %{$BSConfig::unpublishedhook}) {
      if ($prp =~ /^$key/) {
        $unpublish_prp = $key;
        last;
      }
    }
  }
  if ($BSConfig::unpublishedhook && $BSConfig::unpublishedhook->{$unpublish_prp}) {
    my $hook = $BSConfig::unpublishedhook->{$unpublish_prp};
    $hook = [ $hook ] unless ref $hook;
    print "    calling unpublished hook @$hook\n";
    qsystem(@$hook, $prp, $extrep, @db_deleted) && warn("    @$hook failed: $?\n");
  }

  # support for regex usage in $BSConfig::publishedhook
  my $publish_prp = $prp;
  if ($BSConfig::publishedhook_use_regex || $BSConfig::publishedhook_use_regex) {
    for my $key (sort {$b cmp $a} keys %{$BSConfig::publishedhook}) {
      if ($prp =~ /^$key/) {
        $publish_prp = $key;
        last;
      }
    }
  }
  if ($BSConfig::publishedhook && $BSConfig::publishedhook->{$publish_prp}) {
    my $hook = $BSConfig::publishedhook->{$publish_prp};
    $hook = [ $hook ] unless ref $hook;
    print "    calling published hook @$hook\n";
    qsystem(@$hook, $prp, $extrep, @changed) && die("    @$hook failed: $?\n");
  }

  BSNotify::notify('REPO_PUBLISHED', { project => $projid , 'repo' => $repoid, 'buildid' => $publishid});

  # all done. till next time...
  if ($BSConfig::publishprogram && $BSConfig::publishprogram->{$prp}) {
    exit(0);
  }

  # recurse for dbgsplit
  publish($projid, $repoid, $repoinfo->{'splitdebug'}, $packtrack, $sources) if $repoinfo->{'splitdebug'} && !$dbgsplit;

  if ($packtrack && !$dbgsplit) {
    # update the packtrack cache (and remove the 'id' entry)
    my @newcache;
    for my $pt (values %$packtrack) {
      my $id = delete $pt->{'id'};
      push @newcache, $id, [ map {$pt->{$_ eq 'arch' ? 'binaryarch' : $_}} qw{arch name epoch version release disturl buildtime cpeid} ] if $id;
    }
    $repoinfo = BSUtil::retrieve("$reporoot/$prp/:repoinfo", 1) || {};
    if (%$repoinfo) {
      $repoinfo->{'trackercache'} = \@newcache;
      $repoinfo->{'trackercacheversion'} = $trackercacheversion;
      BSUtil::store("$reporoot/$prp/.:repoinfo", "$reporoot/$prp/:repoinfo", $repoinfo);
      @newcache = ();	# free mem
      undef $repoinfo;	# free mem
    }
  }

  if ($do_packtrack && $packtrack && !$dbgsplit) {
    # send notification
    print "    sending binary release tracking notification\n";
    BSNotify::notify('PACKTRACK', { project => $projid , 'repo' => $repoid }, Storable::nfreeze([ map { $packtrack->{$_} } sort keys %$packtrack ]));
  }

  if ($do_sourcepublish && $sources && %$sources && !$dbgsplit) {
    print "    sending source publish notification\n";
    BSNotify::notify('sourcepublish', { project => $projid , 'repo' => $repoid }, BSUtil::tostorable($sources));
  }

  # update repoinfo with status
  if (!$dbgsplit && $state ne '') {
    $repoinfo = BSUtil::retrieve("$reporoot/$prp/:repoinfo", 1) || {};
    $repoinfo->{'code'} = 'succeeded';
    $repoinfo->{'starttime'} = $starttime;
    $repoinfo->{'endtime'} = time();
    $repoinfo->{'publishid'} = $publishid;
    BSUtil::store("$reporoot/$prp/.:repoinfo", "$reporoot/$prp/:repoinfo", $repoinfo);
  }
  return 'succeeded';
}

sub clear_repoinfo_state {
  my ($prp, $details) = @_;
  if (-s "$reporoot/$prp/:repoinfo") {
    my $repoinfo = BSUtil::retrieve("$reporoot/$prp/:repoinfo", 1) || {};
    if ($repoinfo->{'state'} || ($repoinfo->{'code'} || '') eq 'building') {
      delete $repoinfo->{'state'};
      if (($repoinfo->{'code'} || '') eq 'building') {
	delete $repoinfo->{'details'};
        $repoinfo->{'code'} = 'failed';
        $repoinfo->{'endtime'} = time();
        $repoinfo->{'details'} = $details if $details;
      }
      BSUtil::store("$reporoot/$prp/.:repoinfo$$", "$reporoot/$prp/:repoinfo", $repoinfo);
    }
  }
}

# check if background publish is still running
sub check_publish_prg_running {
  my ($prp) = @_;
  return 0 unless $BSConfig::publishprogram && $BSConfig::publishprogram->{$prp};
  local *PPLOCK;
  if (open(PPLOCK, '<', "$reporoot/$prp/.pplock")) {
    if (flock(PPLOCK, LOCK_EX | LOCK_NB)) {
      close PPLOCK;
      return 1;
    }
    close PPLOCK;
  }
  return 0;
}

my %publish_retry;
my %publish_lasttime;
my %publish_duration;

sub writepublishtimeevent {
  my ($projid, $repoid, $now, $duration, $evn) = @_;
  my $ev = {
    'type' => 'publishtime',
    'project' => $projid,
    'repository' => $repoid,
    'job' => "$now $duration",
  };   
  $ev->{'job'} .= " $evn" if defined $evn;
  my $evname = "publishtime::${projid}::$repoid";
  $evname = "publishtime::".Digest::MD5::md5_hex($evname) if length($evname) > 200; 
  writexml("$myeventdir/.$evname$$", "$myeventdir/$evname", $ev, $BSXML::event);
  BSUtil::ping("$myeventdir/.ping");
}

sub clean_publish_retry {
  my $now = time();
  for (sort keys %publish_retry) {
    delete($publish_retry{$_}) if $publish_retry{$_} < $now;
  }
}

sub set_publish_retry {
  my ($req, $due) = @_;
  $publish_retry{$req->{'event'}} = $due;
  my $ev = $req->{'ev'};
  if ($req->{'forked'}) {
    writepublishtimeevent($ev->{'project'}, $ev->{'repository'}, 0, $due, $req->{'event'});
  } else {
    BSUtil::ping("$myeventdir/.ping");	# see BSStdRunnder::setdue
  }
}

sub notify_state {
  my ($projid, $repoid, $state) = @_;
  BSNotify::notify('REPO_PUBLISH_STATE', { 'project' => $projid, 'repo' => $repoid, 'state' => $state });
}

sub syncdb {
  die if @db_sync;
  db_pickup();
  db_sync();
}

sub publishstats {
  my ($req, $prp, $code, $starttime, $endtime) = @_;
  my $ev = $req->{'ev'};
  return unless $ev;
  my $createtime = $ev->{'time'} || $starttime;
  my $flavor = $req->{'flavor'} || '-';
  $flavor =~ s/\s/_/g;
  $code ||= 'unknown';
  print "publish statistics: $prp/- $code $createtime-$starttime-$endtime $flavor\n";
}

sub publishevent {
  my ($req, $projid, $repoid) = @_;

  if ($req->{'forked'})  {
    my $prepend = "$$: ";
    $prepend = "$req->{'flavor'}.$prepend" if $req->{'flavor'};
    binmode STDOUT, "via(BSStdRunner::prepend)";
    print $prepend;
  }

  $db_sync_append = 1 if $req->{'forked'};
  syncdb() if $req->{'syncdb'};

  my $prp = "$projid/$repoid";
  my $starttime = time();

  if (check_publish_prg_running($prp)) {
    set_publish_retry($req, $starttime + 60);
    die("$prp: external publish program still running\n");
  }
  my $penalty_multiplier = defined($BSConfig::publish_penalty_multiplier) ? $BSConfig::publish_penalty_multiplier : 1;
  if ($penalty_multiplier && $publish_lasttime{$prp} && $publish_duration{$prp} > 300 && $publish_lasttime{$prp} + $publish_duration{$prp} * $penalty_multiplier> $starttime) {
    set_publish_retry($req, $starttime + 60 * 5);
    die("$prp: not yet\n");
  }

  notify_state($projid, $repoid, 'publishing');
  my $code;
  eval {
    $code = publish($projid, $repoid);
  };
  if ($@) {
    my $now = time();
    $code = 'failed';
    warn("publish failed for $projid/$repoid : $@");
    # delete state from repoinfo so that we will always re-publish
    my $err = $@;
    $err =~ s/\n.*//s;
    clear_repoinfo_state($prp, $err);
    set_publish_retry($req, $now + 60);
    publishstats($req, $prp, $code, $starttime, $now);
    db_sync();
    BSUtil::printlog("publish failed for $prp") if $req->{'forked'};
    return 0;
  }

  my $now = time();
  $publish_lasttime{$prp} = $now;
  $publish_duration{$prp} = $now - $starttime;
  writepublishtimeevent($projid, $repoid, $now, $now - $starttime) if $req->{'forked'};
  publishstats($req, $prp, $code, $starttime, $now);
  notify_state($projid, $repoid, 'published');
  db_sync();
  BSUtil::printlog("publish done for $prp") if $req->{'forked'};
  return 1;
}

sub publishtimeevent {
  my ($req, $projid, $repoid, $job) = @_;
  my $prp = "$projid/$repoid";
  my ($lasttime, $duration, $evn) = split(' ', $job, 3);
  if ($lasttime) {
    $publish_lasttime{$prp} = $lasttime;
    $publish_duration{$prp} = $duration;
  } elsif (defined($evn)) {
    $publish_retry{$evn} = $duration;
  }
  return 1;
}

sub configurationevent {
  my ($req) = @_;
  print "updating configuration\n";
  BSConfiguration::update_from_configuration();
  return 1;
}

sub lsevents {
  clean_publish_retry() if %publish_retry;
  my @events = BSStdRunner::lsevents(@_);
  # put publishtime and highprio events to the front
  my @publishtimeevents = grep {/^publishtime/} @events;
  @events = grep {!/^publishtime/} @events if @publishtimeevents;
  my @highprioevents = grep {/^_/} @events;
  @events = grep {!/^[_]/} @events if @highprioevents;
  return (@publishtimeevents, @highprioevents, @events);
}

sub getevent {
  my ($req) = @_;
  my $evname = $req->{'event'};
  if ($publish_retry{$evname}) {
    return (undef, 1) if $publish_retry{$evname} > time();
    delete $publish_retry{$evname};
  }
  $req = BSStdRunner::getevent($req);
  return undef unless $req;

  # publishtime and configuration events are nofork events
  my $evtype = $req->{'ev'}->{'type'} || '';
  if ($evtype eq 'publishtime') {
    return ($req, 1, 1) if ($req->{'ev'}->{'job'} || '') =~ /^0 /;
    return ($req, undef, 1);
  }
  return ($req, undef, 1) if $evtype eq 'configuration';

  # also don't fork if the database is not in sync
  if (@db_sync) {
    print "not forking because of unsynced database entries\n";
    return ($req, undef, 1);
  }

  # serialize if our children have unsynced data
  if (!@db_sync && $maxchild && $maxchild > 1 && $extrepodb && -s "$extrepodb.sync") {
    print "not forking because of unsynced database entries (syncdb)\n";
    $req->{'syncdb'} = 1;
    return ($req, undef, 2);
  }

  return $req;
}

sub runsingleevent {
  my ($conf, $eventfile) = @_;

  my $ev = readxml($eventfile, $BSXML::event);
  $conf->{'eventdir'} = '.';
  ($conf->{'eventdir'}, $eventfile) = ($1, $2) if $eventfile =~ /^(.*)\/([^\/]*)$/;
  my $req = {'conf' => $conf, 'event' => $eventfile, 'ev' => $ev};
  my $r = BSStdRunner::dispatch($req);
  exit($r ? 0 : 1);
}

sub call_run {
  my ($conf) = @_;
  db_sync();	# try to sync old events
  runsingleevent($conf, $ARGV[1]) if @ARGV > 1 && $ARGV[0] eq '--event';
  BSRunner::run($conf);
}

sub fc_rescan {
  my ($conf, $fc) = @_;
  unlink($fc);
}
  
my $dispatches = [ 
  'configuration' => \&configurationevent,
  'publish $project $repository' => \&publishevent,
  'publishtime $project $repository $job' => \&publishtimeevent,
];

my $conf = { 
  'runname' => 'bs_publish',
  'eventdir' => $myeventdir,
  'dispatches' => $dispatches,
  'lsevents' => \&lsevents,
  'getevent' => \&getevent,
  'run' => \&call_run,
  'filechecks' => { "$rundir/bs_publish.rescan" => \&fc_rescan },
  'inprogress' => 1,
  'maxchild' => $maxchild,
  'maxchild_flavor' => $maxchild_flavor,
};
$conf->{'getflavor'} = $BSConfig::publish_getflavor if $BSConfig::publish_getflavor;

BSStdRunner::run('publisher', \@ARGV, $conf);

