![[site community profile]](https://www.dreamwidth.org/img/comm_staff.png){kind=small}
[dw-free] import queue viewer admin tool
[commit: http://hg.dwscoalition.org/dw-free/rev/40273080754d]
http://bugs.dwscoalition.org/show_bug.cgi?id=4067
Admin tool for a rough overview of the import queue. Includes a drill-down
to a more detailed per-user view.
Patch by![[personal profile]](https://www.dreamwidth.org/img/silk/identity/user.png)
fu.
Files modified:
http://bugs.dwscoalition.org/show_bug.cgi?id=4067
Admin tool for a rough overview of the import queue. Includes a drill-down
to a more detailed per-user view.
Patch by
fu.Files modified:
- cgi-bin/DW/Controller/Admin/Importer.pm
- cgi-bin/DW/Logic/Importer.pm
- views/admin/importer.tt
- views/admin/importer.tt.text
- views/admin/importer/detail.tt
- views/admin/importer/detail.tt.text
--------------------------------------------------------------------------------
diff -r a286d2a201aa -r 40273080754d cgi-bin/DW/Controller/Admin/Importer.pm
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cgi-bin/DW/Controller/Admin/Importer.pm Fri Nov 25 22:51:19 2011 +0800
@@ -0,0 +1,94 @@
+#!/usr/bin/perl
+#
+# DW::Controller::Importer
+#
+# This controller is to view details about the import queue
+#
+# Authors:
+# Afuna <coder.dw@afunamatata.com>
+#
+# Copyright (c) 2011 by Dreamwidth Studios, LLC.
+#
+# This program is free software; you may redistribute it and/or modify it under
+# the same terms as Perl itself. For a copy of the license, please reference
+# 'perldoc perlartistic' or 'perldoc perlgpl'.
+#
+
+package DW::Controller::Importer;
+
+use strict;
+
+use DW::Controller;
+use DW::Routing;
+use DW::Template;
+use DW::Controller::Admin;
+
+use DW::Logic::Importer;
+
+DW::Routing->register_string( "/admin/importer/index", \&index_controller );
+DW::Routing->register_string( "/admin/importer/details/index", \&detail_controller );
+
+DW::Controller::Admin->register_admin_page( '/',
+ path => 'importer/',
+ ml_scope => '/admin/importer.tt',
+ privs => [ 'siteadmin:theschwartz' ]
+);
+
+sub index_controller {
+ my ( $ok, $rv ) = controller( privcheck => [ "siteadmin:theschwartz" ] );
+ return $rv unless $ok;
+
+ my $vars = {};
+ my $sclient = LJ::theschwartz();
+ my @joblist = $sclient->list_jobs( { funcname => [qw( DW::Worker::ContentImporter::LiveJournal::Bio
+ DW::Worker::ContentImporter::LiveJournal::Tags
+ DW::Worker::ContentImporter::LiveJournal::Entries
+ DW::Worker::ContentImporter::LiveJournal::Comments
+ DW::Worker::ContentImporter::LiveJournal::Userpics
+ DW::Worker::ContentImporter::LiveJournal::Friends
+ DW::Worker::ContentImporter::LiveJournal::FriendGroups
+ DW::Worker::ContentImporter::LiveJournal::Verify
+ )] } );
+
+ my @latest;
+ my @jobs;
+ foreach my $job ( @joblist ) {
+ my $funcname = $job->funcname;
+ my $arg = $job->arg;
+ my $u = LJ::load_userid( $arg->{userid} );
+ my $latest_id = DW::Logic::Importer->get_import_data_for_user( $u )->[0]->[0];
+
+ push @latest, [ $u, $latest_id ];
+
+ push @jobs, { type => $funcname,
+ user => $u->ljuser_display,
+ username => $u->username,
+ importid => { job => $arg->{import_data_id}, latest => $latest_id },
+ };
+ }
+ $vars->{jobs} = \@jobs;
+
+ return DW::Template->render_template( 'admin/importer.tt', $vars );
+}
+
+sub detail_controller {
+ my ( $ok, $rv ) = controller( privcheck => [ "siteadmin:theschwartz" ] );
+ return $rv unless $ok;
+
+ my $get = DW::Request->get->get_args;
+ my $user = $get->{user};
+
+ my $u = LJ::load_user( $user );
+ return error_ml( "error.invaliduser" ) unless $u;
+
+ my $import_items = DW::Logic::Importer->get_queued_imports( $u );
+ my $vars = { username => $u->ljuser_display, import_items => $import_items };
+
+ if ( scalar keys %{$import_items||{}} > 1 ) {
+ $vars->{errmsg} = ".error.toomanypending";
+ }
+
+ return DW::Template->render_template( 'admin/importer/detail.tt', $vars );
+}
+
+1;
diff -r a286d2a201aa -r 40273080754d cgi-bin/DW/Logic/Importer.pm
--- a/cgi-bin/DW/Logic/Importer.pm Wed Nov 23 21:40:13 2011 +0800
+++ b/cgi-bin/DW/Logic/Importer.pm Fri Nov 25 22:51:19 2011 +0800
@@ -21,6 +21,41 @@
use DW::Pay;
use Storable;
+=head1 API
+
+=head2 C<<DW::Logic::Importer->get_import_data( $u, id1, [ id2, id3 ] )>>
+
+Get import data for this user for all provided import ids
+
+=cut
+sub get_import_data {
+ my ( $class, $u, @ids ) = @_;
+
+ my $dbh = LJ::get_db_writer()
+ or die "No database.";
+
+ my $qs = join ",", map { "?" } @ids;
+
+ # FIXME: memcache this
+ my $imports = $dbh->selectall_arrayref(
+ "SELECT import_data_id, hostname, username, usejournal, password_md5, options FROM import_data WHERE userid = ? AND import_data_id IN ( $qs ) " .
+ "ORDER BY import_data_id ASC",
+ undef, $u->id, @ids
+ );
+
+ foreach my $import ( @{$imports||[]} ) {
+ $import->[5] = Storable::thaw( $import->[5] ) || {}
+ if $import->[5];
+ }
+
+ return $imports;
+}
+
+=head2 C<<DW::Logic::Importer->get_import_data_for_user>>
+
+Get the latest import data for this user
+
+=cut
sub get_import_data_for_user {
my ( $class, $u ) = @_;
@@ -41,35 +76,91 @@
return $imports;
}
+=head2 C<<DW::Logic::Importer->get_import_items( $u, id1, [ id2, id3... ] )>>
+
+Get import items for this user for all provided import ids.
+
+=cut
+sub get_import_items {
+ my ( $class, $u, @importids ) = @_;
+
+ my $dbh = LJ::get_db_writer()
+ or die "No database.";
+
+ my $qs = join ",", map { "?" } @importids;
+
+ my $import_items = $dbh->selectall_arrayref(
+ "SELECT import_data_id, item, status, created, last_touch FROM import_items WHERE userid = ? AND import_data_id IN ( $qs )",
+ undef, $u->id, @importids
+ );
+
+ my %ret;
+ foreach my $import_item ( @$import_items ) {
+ my ( $id, $item, $status, $created, $last_touch ) = @$import_item;
+ $ret{$id}->{$item} = {
+ status => $status,
+ created => $created,
+ last_touch => $last_touch,
+ };
+ }
+
+ return \%ret;
+}
+
+=head2 C<<DW::Logic::Importer->get_import_items_for_user( $u, id1, [ id2, id3... ] )>>
+
+Get latest import item for this user. Includes import data.
+
+=cut
+
sub get_import_items_for_user {
my ( $class, $u ) = @_;
my $imports = DW::Logic::Importer->get_import_data_for_user( $u );
my %items;
- my $dbh = LJ::get_db_writer()
- or die "No database.";
my $has_items = 0;
foreach my $import ( @$imports ) {
my ( $importid, $host, $username, $usejournal, $password ) = @$import;
- $items{$importid} = { host => $host, user => $username, pw => $password, usejournal => $usejournal, items => {} };
+ $items{$importid} = {
+ host => $host,
+ user => $username,
+ pw => $password,
+ usejournal => $usejournal,
+ items => DW::Logic::Importer->get_import_items( $u, $importid )->{$importid}, };
- my $import_items = $dbh->selectall_arrayref(
- 'SELECT item, status, created, last_touch FROM import_items WHERE userid = ? AND import_data_id = ?',
- undef, $u->id, $importid
- );
-
- foreach my $import_item ( @$import_items ) {
- $items{$importid}->{items}->{$import_item->[0]} = {
- status => $import_item->[1], created => $import_item->[2], last_touch => $import_item->[3]
- };
- $has_items = 1 if $import_item->[0];
- }
+ $has_items = 1 if scalar keys %{ $items{$importid}->{items} };
}
return $has_items ? \%items : {};
}
+=head2 C<<DW::Logic::Importer->get_queued_imports( $u )>>
+
+Get a list of imports that have yet to be processed. May be in the schwartz queue, and thus running soon
+or else in the import queue, and have yet to be put into the schwartz queue. The latter may not run if they
+are duplicates of something in the schwartz queue.
+
+=cut
+
+sub get_queued_imports {
+ my ( $class, $u ) = @_;
+
+ return {} unless $u;
+
+ # the latest import the user has queued
+ my $latestimport = DW::Logic::Importer->get_import_data_for_user( $u );
+
+ # the latest job that's currently running
+ # should be <= $latestimport
+ my $runningjob = $u->prop( "import_job" );
+
+ return {} unless $latestimport && $runningjob;
+
+ return DW::Logic::Importer->get_import_items( $u, $runningjob .. $latestimport->[0]->[0] );
+}
+
+
sub set_import_data_for_user {
my ( $class, $u, %opts ) = @_;
diff -r a286d2a201aa -r 40273080754d views/admin/importer.tt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/views/admin/importer.tt Fri Nov 25 22:51:19 2011 +0800
@@ -0,0 +1,35 @@
+[%# admin/importer.tt
+
+Admin page for the importer queue
+
+Authors:
+ Afuna <coder.dw@afunamatata.com>
+
+This program is free software; you may redistribute it and/or modify it under
+the same terms as Perl itself. For a copy of the license, please reference
+'perldoc perlartistic' or 'perldoc perlgpl'.
+%]
+[%- sections.title = '.title' | ml -%]
+
+<h2>[% '.theschwartz.header' | ml %]</h2>
+<p>[% '.theschwartz.intro' | ml %]</p>
+<p>[% '.theschwartz.queue' | ml( num = jobs.size ) %]</p>
+
+
+[%- IF jobs.size > 0 -%]
+<table>
+<thead><tr><th>Order</th><th>User</th><th>Job Type</th><th>Job Import ID</th><th>Latest Queued ID</th><th>Details</th></tr>
+<tbody>
+ [%- FOREACH job = jobs -%]
+ <tr>
+ <td>[% loop.count %]</td>
+ <td>[% job.user %]</td>
+ <td>[% job.type %]</td>
+ <td>[% job.importid.job %]</td>
+ <td>[% job.importid.latest %]</td>
+ <td><a href="[% site.root %]/admin/importer/details/?user=[% job.username %]">Details</a></td>
+ </tr>
+ [%- END -%]
+</tbody>
+</table>
+[%- END -%]
diff -r a286d2a201aa -r 40273080754d views/admin/importer.tt.text
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/views/admin/importer.tt.text Fri Nov 25 22:51:19 2011 +0800
@@ -0,0 +1,12 @@
+;; -*- coding: utf-8 -*-
+.admin.link=Importer Queue
+
+.admin.text=View importer queue status
+
+.theschwartz.header=TheSchwartz Queue
+
+.theschwartz.intro=This is a list of jobs currently in the (schwartz) job queue, but does not include jobs that are waiting for other things to complete (e.g., comment imports that are still importing comments).
+
+.theschwartz.queue=There [[?num|is|are]] [[num]] [[?num|item|items]] in the queue.
+
+.title=Importer Queue
diff -r a286d2a201aa -r 40273080754d views/admin/importer/detail.tt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/views/admin/importer/detail.tt Fri Nov 25 22:51:19 2011 +0800
@@ -0,0 +1,46 @@
+[%# admin/importer/detail.tt
+
+Details for pending imports for a user
+
+Authors:
+ Afuna <coder.dw@afunamatata.com>
+
+This program is free software; you may redistribute it and/or modify it under
+the same terms as Perl itself. For a copy of the license, please reference
+'perldoc perlartistic' or 'perldoc perlgpl'.
+%]
+[%- sections.title = '.title' | ml -%]
+
+[%- USE date -%]
+
+Pending imports for [% username %].
+
+[% IF errmsg %]
+<div class='error-box'>
+ <p>[% errmsg | ml %]</p>
+</div>
+[% END %]
+
+[%- IF import_items.size > 0 -%]
+ [%- FOREACH import_id = import_items.keys.sort -%]
+ <h3>Import #[% import_id %]</h3>
+ <table>
+ <thead><tr>
+ <th>Item</th>
+ <th>Status</th>
+ <th>Created</th>
+ <th>Last Touch</th>
+ </tr></thead>
+ <tbody>
+ [%- FOREACH item = import_items.$import_id.pairs -%]
+ <tr>
+ <td>[% item.key %]</td>
+ <td>[% item.value.status %]</td>
+ <td>[% date.format( item.value.created ) %]</td>
+ <td>[% item.value.last_touch ? date.format( item.value.last_touch ) : "-" %]</td>
+ </tr>
+ [%- END -%]
+ </tbody>
+ </table>
+ [%- END %]
+[%- END -%]
diff -r a286d2a201aa -r 40273080754d views/admin/importer/detail.tt.text
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/views/admin/importer/detail.tt.text Fri Nov 25 22:51:19 2011 +0800
@@ -0,0 +1,4 @@
+;; -*- coding: utf-8 -*-
+.error.toomanypending=There are too many pending imports. *Probably* what is happening here is that the user's import is already in the schwartz async queue where we can't look it up easily, but then they started a new import. The old import will still keep running, and the new import will show up as aborted.
+
+.title=Pending Import Details
--------------------------------------------------------------------------------