Refbase2X/contrib/command_line/refbase_import

#!/usr/bin/perl

# Project:    Web Reference Database (refbase) <http://www.refbase.net>
# Copyright:  Matthias Steffens <mailto:refbase@extracts.de> and the file's
#             original author(s).
#
#             This code is distributed in the hope that it will be useful,
#             but WITHOUT ANY WARRANTY. Please see the GNU General Public
#             License for more details.
#
# File:       ./contrib/command_line/refbase_import
# Repository: $HeadURL: file:///svn/p/refbase/code/branches/bleeding-edge/contrib/command_line/refbase_import $
# Author(s):  Matthias Steffens <mailto:refbase@extracts.de>
#
# Created:    06-Jun-06, 18:00
# Modified:   $Date: 2008-10-30 18:16:54 +0000 (Thu, 30 Oct 2008) $
#             $Author: msteffens $
#             $Revision: 1289 $

# REFBASE_IMPORT -- a refbase command line interface

# Purpose:    Perl script that allows to upload data in various formats to a refbase online database from the command line
# Usage:      refbase_import [OPTIONS] [FILE]

# Help:       For help with the syntax type 'refbase_import -h'
#             To view some usage examples type 'refbase_import -X'
#             Further information is available at <http://cli.refbase.net/>
#             A list of supported import formats is given at <http://import.refbase.net/>

# Version:    1.2

# Requires:   - a shell with Perl execution capabilities
#             - the Perl CPAN modules LWP::UserAgent, HTTP::Request::Common, HTTP::Response, HTTP::Cookies and URI::URL
#             - access with import permissions to a refbase database (refbase-0.9.0 or greater)

# Limits:     - The character encoding of your import data must match the encoding of your refbase database (i.e., 'latin1' or 'utf8')
#             - The authentication mechanism is currently limited in that a given password will be transferred as parameter in the POST request

# --------------------------------------------------------------------------------------------------------------

$version = "1.2";

# Configure variables:

# Specify the full URLs to any refbase servers that shall be queried:
# Notes: - the given hash keys will work as shortcuts, e.g. '--host=local' would upload
#          data to your local refbase installation; one hash key must be named 'default',
#          all other keys can be freely chosen
#        - by default, data will be uploaded to the server labeled with key 'default'
%hosts = (
				'default' => 'http://beta.refbase.net/',
				'local'   => 'http://localhost/refs/',
				'beta'    => 'http://beta.refbase.net/',
				'beta2'   => 'http://refbase.textdriven.com/beta/',
				'demo'    => 'http://demo.refbase.net/',
				'org'     => 'http://www.refbase.org/'
);

# Specify the default values for all options that are not explicitly specified:
%params = (
				# import options:
				'skipBadRecords'      => '0',      # -b|--skipbad    -> must be '0' (don't skip records with unrecognized data format) or '1' (skip records with unrecognized data format)
				'importRecordsRadio'  => 'all',    # -i|--import     -> must be 'all' (import all records) or 'only' (import only those records specified in 'importRecords')
				'sourceIDs'           => '',       # -p|--pmid       -> this also applies for '--arxiv|--doi|--openurl' since they are essentially just aliases for '-p'
				'importRecords'       => '1',      # -r|--records    -> must be a list of numbers and/or ranges (e.g., '1-5' will import the first five records; '1 3-5 7' will import records 1, 3, 4, 5 and 7)
				'formType'            => 'data',   # -t|--type       -> must be 'data' (generic data import) or 'id' (import via ID)

				# fixed parameters:
				'client'              => "cli-refbase_import-" . $version # the client ID of this command line utility
);

%outputParams = (
				# output options:
				'citeStyle'           => '',        # -C|--style     => desired citation style, given name must match an entry within the database's MySQL table 'styles' (keep empty to use the database default)
				'format'              => 'ascii',   # -F|--format    => output format must be 'html', 'rtf', 'pdf', 'latex', 'latex_bbl', 'markdown', 'ascii', 'ads', 'bibtex', 'endnote', 'isi', 'ris', 'atom', 'mods', 'oai_dc', 'odf', 'srw_dc', 'srw_mods', 'word' or '' (the empty string '' will produce the default 'ascii' output style)
				'showLinks'           => '1',       # -L|--showlinks => hide/display links column in HTML output; must be '0', '1', or '' (the empty string '' will produce the default output style, i.e. print any links)
				'citeOrder'           => 'author',  # -O|--order     => cite order must be 'author', 'year', 'type', 'type-year', 'creation-date' or '' (the empty string '' will produce the default 'author' sort order)
				'viewType'            => 'web'      # -V|--view      => view type of HTML output; must be 'Web', 'Print', 'Mobile' or '' (the empty string '' will produce the default 'Web' output style)
);

# Specify the default login credentials for a refbase user account:
# Imported data will get associated with this user account
%loginParams = (
				'loginEmail'     => '', # -U|--user     -> the login email address of an existing refbase user with import permissions
				'loginPassword'  => ''  # -P|--password -> the password for the given user account
);

# Specify the location of the cookie jar file:
# This file will be used to store & retrieve cookies
$cookieJarFile = "$ENV{HOME}/.lwpcookies.txt";

# --------------------------------------------------------------------------------

# NOTE: You shouldn't need to change anything below this line

# CPAN modules:
use LWP::UserAgent; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/LWP/UserAgent.pm>
use HTTP::Request::Common; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/HTTP/Request/Common.pm>
use HTTP::Response; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/HTTP/Response.pm>
use HTTP::Cookies; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/HTTP/Cookies.pm>
use URI::URL; # more info: <http://search.cpan.org/~gaas/URI-1.35/URI/URL.pm>

# initialize variables:
$host = $hosts{'default'};
$format = '';

# Extract options:
# TODO: use Getopt::Long

# general options:
if (($ARGV[0] eq '--help') or ($ARGV[0] eq '-h') or ($ARGV[0] eq '')) { &usage (0); } # if the user asked for --help/-h or didn't provide any input, call the 'usage' subroutine
elsif (($ARGV[0] eq '--version') or ($ARGV[0] eq '-v')) { &version (0); } # show version information
elsif (($ARGV[0] eq '--examples') or ($ARGV[0] eq '-X')) { &examples (0); } # print some usage examples

else {
	foreach (@ARGV) {
		# extract import options:
		if ($_ =~ /^(?:-b|--skipbad)=(.+)$/) { $params{'skipBadRecords'} = $1; }
		elsif ($_ =~ /^(?:-i|--import)=(.+)$/) { $params{'importRecordsRadio'} = $1; }
		elsif ($_ =~ /^(?:-p|--pmid|--arxiv|--doi|--openurl)=(.+)$/) { $params{'sourceIDs'} = $1; }
		elsif ($_ =~ /^(?:-r|--records)=(.+)$/) { $params{'importRecords'} = $1; }
		elsif ($_ =~ /^(?:-t|--type)=(.+)$/) { $params{'formType'} = $1; }

		# extract output options:
		elsif ($_ =~ /^(?:-C|--style)=(.+)$/) { $outputParams{'citeStyle'} = $1; }
		elsif ($_ =~ /^(?:-F|--format)=(.+)$/) { $outputParams{'format'} = $1; }
		elsif ($_ =~ /^(?:-L|--showlinks)=(.+)$/) { $outputParams{'showLinks'} = $1; }
		elsif ($_ =~ /^(?:-O|--order)=(.+)$/) { $outputParams{'citeOrder'} = $1; }
		elsif ($_ =~ /^(?:-V|--view)=(.+)$/) { $outputParams{'viewType'} = $1; }

		# extract server options:
		elsif ($_ =~ /^(?:-H|--host)=(.+)$/) { $host = $1; }
		elsif ($_ =~ /^(?:-P|--password)=(.+)$/) { $loginParams{'loginPassword'} = $1; }
		elsif ($_ =~ /^(?:-U|--user)=(.+)$/) { $loginParams{'loginEmail'} = $1; }

		# extract file:
		# (note that if multiple files were given, only the last given file will be honoured)
		elsif ($_ =~ /^(?!(-[biprtCFLOVHPU]|--(?:skipbad|import|pmid|arxiv|doi|openurl|records|type|style|format|showlinks|order|view|host|password|user))=)([^ ]+)/) { @sourceFile = $2; }
	}
}

# for '--type=data', check if a source file was specified:
if (($params{'formType'} =~ /^data$/i) && (scalar @sourceFile == 0)) {
	print "There were validation errors regarding the data you submitted:\n\n";
	print "FILE: The file operand is missing! The generic data import feature ('--type=data')\n"
		. "      requires a FILE to be specified. Type 'refbase_import -X' to see some usage\n"
		. "      examples. For general help with the syntax type 'refbase_import -h'.\n\n";
	exit;
}
# for '--type=id' (or, previously: --type=pmid), check if at least one PubMed ID, arXiv ID, DOI or OpenURL was given:
# TODO: improve identification/verification of the given IDs
elsif (($params{'formType'} =~ /^(pm)?id$/i) && ($params{'sourceIDs'} !~ /\d+/)) {
	print "There were validation errors regarding the data you submitted:\n\n";
	print "sourceIDs: You must specify at least one PubMed ID, arXiv ID, DOI or OpenURL! The 'import via ID'\n"
		. "           feature ('--type=id') requires the '-p, --pmid' option or one of '--arxiv|--doi|--openurl'\n"
		. "           to be specified. Type 'refbase_import -X' to see some usage examples. For general help\n"
		. "           with the syntax type 'refbase_import -h'.\n\n";
	exit;
}

# adjust form type value:
if ($params{'formType'} =~ /^(pm)?id$/i) { # --type=id (or, previously: --type=pmid)
	$params{'formType'} = "importID";
}
else { # --type=data
	$params{'formType'} = "import";
}

# resolve any host shortcuts:
if (exists($hosts{$host})) {
	$host = $hosts{$host};
}
elsif ($host !~ /^https?:\/\//i) {
	$host = $hosts{'default'}; # can't resolve given host, reset back to default
}

# assign correct URL params based on the '-F|--format' option:
if (exists($outputParams{'format'})) {
	$format = $outputParams{'format'};
	if ($format =~ /^(html|rtf|pdf|latex|latex_bbl|markdown|ascii)$/i) {
		$outputParams{'submit'} = "Cite";
	}
	if ($format =~ /^(html|rtf|pdf|latex|latex_bbl|markdown|ascii)$/i) {
		$format =~ s/^latex_bbl$/LaTeX .bbl/i;
		$outputParams{'citeType'} = $format;
	}
	elsif ($format =~ /^(ads|bibtex|endnote|isi|ris|atom|mods|oai_dc|odf|srw(_dc|_mods)?|word)$/i) {
		$outputParams{'submit'} = "Export";
		$outputParams{'exportType'} = "file";

		if ($format =~ /^(ads|bibtex|endnote|isi|ris)$/i) {
			$outputParams{'exportFormat'} = $format;
		}
		elsif ($format =~ /^(atom|mods|oai_dc|odf|srw(_dc|_mods)?|word)$/i) {
			$outputParams{'exportFormat'} = $format . " xml";
		}
	}
	else {
		$outputParams{'citeType'} = "ascii";
	}

	delete($outputParams{'format'});
}

# construct URL:
# (uses URI::URL)
$importScript = "import_modify.php";
$importURL = url($host . $importScript);

# initialize new user agent:
# (uses LWP::UserAgent)
$userAgent = LWP::UserAgent->new;

# set user agent string:
$userAgent->agent("refbase_import/" . $version . " (http://cli.refbase.net/) ");

# set cookie jar object:
# LWP will collect cookies and respond to cookie requests via its cookie jar, thus
# enabling the user agent to fetch a PHP session ID from the refbase login response
# and automatically resend it upon next import request
$userAgent->cookie_jar({ file => $cookieJarFile, autosave => 1 });

# attempt to authenticate using the given login credentials:
if (($loginParams{'loginEmail'} ne '') && ($loginParams{'loginPassword'} ne '')) {
	$loginSuccessful = &login(0); # call the 'login' subroutine
}
else {
	$loginSuccessful = 0;
}

if (!$loginSuccessful) {
	print "Login failed! You provided an incorrect email address or password.\n\n";
	exit;
}

# send POST request:
# (uses HTTP::Request::Common & HTTP::Response)
if ($params{'formType'} =~ /^importID$/i) { # --type=id (or, previously: --type=pmid)
	$request = POST $importURL, \%params;
}
else { # --type=data
	$params{'uploadFile'} = \@sourceFile;
	$request = POST $importURL, Content_Type => 'form-data', Content => \%params;
}

$response = $userAgent->request($request);

if ($response->is_error()) {
	print STDERR $response->status_line, "\n";
}
else {
	$location = $response->header('Location');

	if ($location ne '') {
		if ($location =~ /show.php/) {
			# display imported records:
			foreach $key (keys %outputParams) {
				$location .= "&" . $key . "=" . $outputParams{$key};
			}
			if ($location =~ /&headerMsg=\D*(\d+)/i) {
				$location .= "&showRows=" . $1;
			}
		}

		# construct URL:
		# (uses URI::URL)
		$responseURL = url($host . $location);
	
		# send GET request:
		# (uses HTTP::Request::Common & HTTP::Response)
		$request = GET $responseURL;
		$response = $userAgent->request($request); # or use: $response = $userAgent->get($responseURL);
	}

	binmode STDOUT;
	print $response->content();
}

# --------------------------------------------------------------------------------

# Login with login credentials given in '%loginParams':
sub login
{
	local ($status) = @_;

	# construct URL:
	# (uses URI::URL)
	$loginScript = "user_login.php";
	$loginURL = url($host . $loginScript);

	# send POST request:
	# (uses HTTP::Request::Common & HTTP::Response)
	$request = POST $loginURL, \%loginParams;

	$response = $userAgent->request($request);
	
	if ($response->is_error()) {
		print STDERR $response->status_line, "\n";
		exit $status;
	}
	else {
		$location = $response->header('Location');

		# upon successful login, refbase will redirect to 'index.php'
		if ($location =~ /index.php/) {
			return 1; # login successful
		}
		else {
			return 0; # login NOT successful
		}
	}
}

# --------------------------------------------------------------------------------

# Print usage and exit:
sub usage
{
	local ($status) = @_;

	print "\nrefbase_import command line client, v" . $version . " by Matthias Steffens, http://cli.refbase.net/\n\n"

		. "Usage:   refbase_import [OPTIONS] [FILE]\n\n"

		. "Notes:   - Two import modes are supported:\n"
		. "           1) '--type=data' requires an import FILE to be specified;\n"
		. "              for supported import formats, see: http://import.refbase.net/\n"
		. "           2) '--type=id' requires the '-p, --pmid' option or one of '--arxiv|--doi|--openurl' with\n"
		. "              one or more whitespace-delimited PubMed IDs, arXiv IDs, DOIs or OpenURLs, respectively.\n"
		. "         - Options syntax: [OPTION]=[VALUE], e.g. '-p=16351846' or '--pmid=\"16351846 16783713\"'.\n"
		. "         - For each option, default values can be specified at the top of the script.\n"
		. "           Current defaults are given in parentheses.\n\n"

		. "General Options:   -h, --help        - display this help text\n"
		. "                   -v, --version     - display version information\n"
		. "                   -X, --examples    - display usage examples\n\n"

		. "Import Options:    -b, --skipbad     - skip records with unrecognized data format ('" . $params{'skipBadRecords'} . "')\n"
		. "                                       possible values: 0, 1\n"
		. "                   -i, --import      - import all or only some records ('" . $params{'importRecordsRadio'} . "')\n"
		. "                                       possible values: all, only\n"
		. "                   -p, --pmid,       - IDs of records to import ('" . $params{'sourceIDs'} . "')\n"
		. "                       --arxiv,        supported IDs: PubMed ID (PMID), arXiv ID, DOI, OpenURL\n"
		. "                       --doi,          \n"
		. "                       --openurl       \n"
		. "                   -r, --records     - positional numbers and/or ranges of records to import ('" . $params{'importRecords'} . "')\n"
		. "                                       requires the '--import=only' option\n"
		. "                   -t, --type        - import type ('" . $params{'formType'} . "')\n"
		. "                                       possible values: data, id\n\n"

		. "Output Options:    -C, --style       - citation style ('" . $outputParams{'citeStyle'} . "')\n"
		. "                   -F, --format      - output format ('" . $outputParams{'format'} . "')\n"
		. "                                       possible values: html, rtf, pdf, latex, latex_bbl, markdown, ascii,\n"
		. "                                                        ads, bibtex, endnote, isi, ris, atom, mods, oai_dc,\n"
		. "                                                        odf, srw_dc, srw_mods, word\n"
		. "                   -L, --showlinks   - hide/display links column in html output ('" . $outputParams{'showLinks'} . "')\n"
		. "                                       possible values: 0, 1\n"
		. "                   -O, --order       - sort order of returned records ('" . $outputParams{'citeOrder'} . "')\n"
		. "                                       possible values: author, year, type, type-year, creation-date\n"
		. "                   -V, --view        - view type of html output ('" . $outputParams{'viewType'} . "')\n"
		. "                                       possible values: web, print, mobile\n\n"

		. "Server Options:    -H, --host        - URL of the refbase database ('" . $host . "')\n"
		. "                                       defined shortcuts: " . join(', ', sort keys(%hosts)) . "\n"
		. "                   -P, --password    - password for given '-U, --user' account";

	if ($loginParams{'loginPassword'} ne '') {
		print "\n                                       (a default pwd has been defined)\n";
	}
	else {
		print " ('')\n";
	}

	print "                   -U, --user        - login email address of an existing refbase user with\n"
		. "                                       import permissions ('" . $loginParams{'loginEmail'} . "')\n\n";

	exit $status;
}

# --------------------------------------------------------------------------------

# Print version number and exit:
sub version
{
	local ($status) = @_;
	print "\nrefbase_import command line client, version " . $version
		. "\ncheck for updates at http://cli.refbase.net/\n\n";
	exit $status;
}

# --------------------------------------------------------------------------------

# Print examples and exit:
sub examples
{
	local ($status) = @_;
	print <<'END_EXAMPLES';

 --------------------------------------------------------------------------------
 REFBASE_IMPORT USAGE EXAMPLES:
 --------------------------------------------------------------------------------

 1) Import BibTeX records from file 'import.bib' using the defaults defined
    within the refbase_import script:
 
    refbase_import import.bib
 
 --------------------------------------------------------------------------------

 2) Import all Endnote records given in file 'import.enw' into your default
    refbase database:
 
    refbase_import -t=data -i=all import.enw
 
 --------------------------------------------------------------------------------

 3) Take RIS records from file 'import.ris' but import only the first three as
    well as the fifth and the tenth record into your local refbase database:
 
    refbase_import -H=local -t=data -i=only -r=1-3,5,10 import.ris
 
 --------------------------------------------------------------------------------

 4) Import MODS XML records from file 'mods.xml' into the refbase demo database
    using the defaults defined within the refbase_import script:
 
    refbase_import -H=http://demo.refbase.net/ -U=user@refbase.net -P=user mods.xml
 
 --------------------------------------------------------------------------------

 5) Fetch two records from PubMed.gov via their PMID (i.e. the unique PubMed
    identifier, in this example, records with PMIDs 16351846 and 16783713) and
    import them into your local refbase database:
 
    refbase_import -H=local -t=id -p="16351846 16783713"
 
 --------------------------------------------------------------------------------

 6) Fetch two records from CrossRef.org via their DOI (i.e. the unique Document
    Object Identifier, in this example, records with DOIs 10.3354/meps251037 and
    10.1103/PhysRev.47.777) and import them into your local refbase database:
 
    refbase_import -H=local -t=id --doi="10.3354/meps251037 10.1103/PhysRev.47.777"
 
 --------------------------------------------------------------------------------

 7) Fetch three records from arXiv.org via their arXiv ID (i.e. the unique arXiv
    identifier, in this example, records with arXiv IDs astro-ph/0609768, 0806.1829
    and 0802.0204v1) and import them into your local refbase database:
 
    refbase_import -H=local -t=id --arxiv="astro-ph/0609768 0806.1829 0802.0204v1"
 
 --------------------------------------------------------------------------------

END_EXAMPLES
	exit $status;
}

__END__