You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
459 lines
20 KiB
459 lines
20 KiB
#!/usr/bin/perl
|
|
|
|
# Project: Web Reference Database (refbase) <http://www.refbase.net>
|
|
# Copyright: Matthias Steffens <mailto:refbase@extracts.de> and the file's
|
|
# original author(s).
|
|
#
|
|
# This code is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY. Please see the GNU General Public
|
|
# License for more details.
|
|
#
|
|
# File: ./contrib/command_line/refbase_import
|
|
# Repository: $HeadURL: file:///svn/p/refbase/code/branches/bleeding-edge/contrib/command_line/refbase_import $
|
|
# Author(s): Matthias Steffens <mailto:refbase@extracts.de>
|
|
#
|
|
# Created: 06-Jun-06, 18:00
|
|
# Modified: $Date: 2008-10-30 18:16:54 +0000 (Thu, 30 Oct 2008) $
|
|
# $Author: msteffens $
|
|
# $Revision: 1289 $
|
|
|
|
# REFBASE_IMPORT -- a refbase command line interface
|
|
|
|
# Purpose: Perl script that allows to upload data in various formats to a refbase online database from the command line
|
|
# Usage: refbase_import [OPTIONS] [FILE]
|
|
|
|
# Help: For help with the syntax type 'refbase_import -h'
|
|
# To view some usage examples type 'refbase_import -X'
|
|
# Further information is available at <http://cli.refbase.net/>
|
|
# A list of supported import formats is given at <http://import.refbase.net/>
|
|
|
|
# Version: 1.2
|
|
|
|
# Requires: - a shell with Perl execution capabilities
|
|
# - the Perl CPAN modules LWP::UserAgent, HTTP::Request::Common, HTTP::Response, HTTP::Cookies and URI::URL
|
|
# - access with import permissions to a refbase database (refbase-0.9.0 or greater)
|
|
|
|
# Limits: - The character encoding of your import data must match the encoding of your refbase database (i.e., 'latin1' or 'utf8')
|
|
# - The authentication mechanism is currently limited in that a given password will be transferred as parameter in the POST request
|
|
|
|
# --------------------------------------------------------------------------------------------------------------
|
|
|
|
$version = "1.2";
|
|
|
|
# Configure variables:
|
|
|
|
# Specify the full URLs to any refbase servers that shall be queried:
|
|
# Notes: - the given hash keys will work as shortcuts, e.g. '--host=local' would upload
|
|
# data to your local refbase installation; one hash key must be named 'default',
|
|
# all other keys can be freely chosen
|
|
# - by default, data will be uploaded to the server labeled with key 'default'
|
|
%hosts = (
|
|
'default' => 'http://beta.refbase.net/',
|
|
'local' => 'http://localhost/refs/',
|
|
'beta' => 'http://beta.refbase.net/',
|
|
'beta2' => 'http://refbase.textdriven.com/beta/',
|
|
'demo' => 'http://demo.refbase.net/',
|
|
'org' => 'http://www.refbase.org/'
|
|
);
|
|
|
|
# Specify the default values for all options that are not explicitly specified:
|
|
%params = (
|
|
# import options:
|
|
'skipBadRecords' => '0', # -b|--skipbad -> must be '0' (don't skip records with unrecognized data format) or '1' (skip records with unrecognized data format)
|
|
'importRecordsRadio' => 'all', # -i|--import -> must be 'all' (import all records) or 'only' (import only those records specified in 'importRecords')
|
|
'sourceIDs' => '', # -p|--pmid -> this also applies for '--arxiv|--doi|--openurl' since they are essentially just aliases for '-p'
|
|
'importRecords' => '1', # -r|--records -> must be a list of numbers and/or ranges (e.g., '1-5' will import the first five records; '1 3-5 7' will import records 1, 3, 4, 5 and 7)
|
|
'formType' => 'data', # -t|--type -> must be 'data' (generic data import) or 'id' (import via ID)
|
|
|
|
# fixed parameters:
|
|
'client' => "cli-refbase_import-" . $version # the client ID of this command line utility
|
|
);
|
|
|
|
%outputParams = (
|
|
# output options:
|
|
'citeStyle' => '', # -C|--style => desired citation style, given name must match an entry within the database's MySQL table 'styles' (keep empty to use the database default)
|
|
'format' => 'ascii', # -F|--format => output format must be 'html', 'rtf', 'pdf', 'latex', 'latex_bbl', 'markdown', 'ascii', 'ads', 'bibtex', 'endnote', 'isi', 'ris', 'atom', 'mods', 'oai_dc', 'odf', 'srw_dc', 'srw_mods', 'word' or '' (the empty string '' will produce the default 'ascii' output style)
|
|
'showLinks' => '1', # -L|--showlinks => hide/display links column in HTML output; must be '0', '1', or '' (the empty string '' will produce the default output style, i.e. print any links)
|
|
'citeOrder' => 'author', # -O|--order => cite order must be 'author', 'year', 'type', 'type-year', 'creation-date' or '' (the empty string '' will produce the default 'author' sort order)
|
|
'viewType' => 'web' # -V|--view => view type of HTML output; must be 'Web', 'Print', 'Mobile' or '' (the empty string '' will produce the default 'Web' output style)
|
|
);
|
|
|
|
# Specify the default login credentials for a refbase user account:
|
|
# Imported data will get associated with this user account
|
|
%loginParams = (
|
|
'loginEmail' => '', # -U|--user -> the login email address of an existing refbase user with import permissions
|
|
'loginPassword' => '' # -P|--password -> the password for the given user account
|
|
);
|
|
|
|
# Specify the location of the cookie jar file:
|
|
# This file will be used to store & retrieve cookies
|
|
$cookieJarFile = "$ENV{HOME}/.lwpcookies.txt";
|
|
|
|
# --------------------------------------------------------------------------------
|
|
|
|
# NOTE: You shouldn't need to change anything below this line
|
|
|
|
# CPAN modules:
|
|
use LWP::UserAgent; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/LWP/UserAgent.pm>
|
|
use HTTP::Request::Common; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/HTTP/Request/Common.pm>
|
|
use HTTP::Response; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/HTTP/Response.pm>
|
|
use HTTP::Cookies; # more info: <http://search.cpan.org/~gaas/libwww-perl-5.805/lib/HTTP/Cookies.pm>
|
|
use URI::URL; # more info: <http://search.cpan.org/~gaas/URI-1.35/URI/URL.pm>
|
|
|
|
# initialize variables:
|
|
$host = $hosts{'default'};
|
|
$format = '';
|
|
|
|
# Extract options:
|
|
# TODO: use Getopt::Long
|
|
|
|
# general options:
|
|
if (($ARGV[0] eq '--help') or ($ARGV[0] eq '-h') or ($ARGV[0] eq '')) { &usage (0); } # if the user asked for --help/-h or didn't provide any input, call the 'usage' subroutine
|
|
elsif (($ARGV[0] eq '--version') or ($ARGV[0] eq '-v')) { &version (0); } # show version information
|
|
elsif (($ARGV[0] eq '--examples') or ($ARGV[0] eq '-X')) { &examples (0); } # print some usage examples
|
|
|
|
else {
|
|
foreach (@ARGV) {
|
|
# extract import options:
|
|
if ($_ =~ /^(?:-b|--skipbad)=(.+)$/) { $params{'skipBadRecords'} = $1; }
|
|
elsif ($_ =~ /^(?:-i|--import)=(.+)$/) { $params{'importRecordsRadio'} = $1; }
|
|
elsif ($_ =~ /^(?:-p|--pmid|--arxiv|--doi|--openurl)=(.+)$/) { $params{'sourceIDs'} = $1; }
|
|
elsif ($_ =~ /^(?:-r|--records)=(.+)$/) { $params{'importRecords'} = $1; }
|
|
elsif ($_ =~ /^(?:-t|--type)=(.+)$/) { $params{'formType'} = $1; }
|
|
|
|
# extract output options:
|
|
elsif ($_ =~ /^(?:-C|--style)=(.+)$/) { $outputParams{'citeStyle'} = $1; }
|
|
elsif ($_ =~ /^(?:-F|--format)=(.+)$/) { $outputParams{'format'} = $1; }
|
|
elsif ($_ =~ /^(?:-L|--showlinks)=(.+)$/) { $outputParams{'showLinks'} = $1; }
|
|
elsif ($_ =~ /^(?:-O|--order)=(.+)$/) { $outputParams{'citeOrder'} = $1; }
|
|
elsif ($_ =~ /^(?:-V|--view)=(.+)$/) { $outputParams{'viewType'} = $1; }
|
|
|
|
# extract server options:
|
|
elsif ($_ =~ /^(?:-H|--host)=(.+)$/) { $host = $1; }
|
|
elsif ($_ =~ /^(?:-P|--password)=(.+)$/) { $loginParams{'loginPassword'} = $1; }
|
|
elsif ($_ =~ /^(?:-U|--user)=(.+)$/) { $loginParams{'loginEmail'} = $1; }
|
|
|
|
# extract file:
|
|
# (note that if multiple files were given, only the last given file will be honoured)
|
|
elsif ($_ =~ /^(?!(-[biprtCFLOVHPU]|--(?:skipbad|import|pmid|arxiv|doi|openurl|records|type|style|format|showlinks|order|view|host|password|user))=)([^ ]+)/) { @sourceFile = $2; }
|
|
}
|
|
}
|
|
|
|
# for '--type=data', check if a source file was specified:
|
|
if (($params{'formType'} =~ /^data$/i) && (scalar @sourceFile == 0)) {
|
|
print "There were validation errors regarding the data you submitted:\n\n";
|
|
print "FILE: The file operand is missing! The generic data import feature ('--type=data')\n"
|
|
. " requires a FILE to be specified. Type 'refbase_import -X' to see some usage\n"
|
|
. " examples. For general help with the syntax type 'refbase_import -h'.\n\n";
|
|
exit;
|
|
}
|
|
# for '--type=id' (or, previously: --type=pmid), check if at least one PubMed ID, arXiv ID, DOI or OpenURL was given:
|
|
# TODO: improve identification/verification of the given IDs
|
|
elsif (($params{'formType'} =~ /^(pm)?id$/i) && ($params{'sourceIDs'} !~ /\d+/)) {
|
|
print "There were validation errors regarding the data you submitted:\n\n";
|
|
print "sourceIDs: You must specify at least one PubMed ID, arXiv ID, DOI or OpenURL! The 'import via ID'\n"
|
|
. " feature ('--type=id') requires the '-p, --pmid' option or one of '--arxiv|--doi|--openurl'\n"
|
|
. " to be specified. Type 'refbase_import -X' to see some usage examples. For general help\n"
|
|
. " with the syntax type 'refbase_import -h'.\n\n";
|
|
exit;
|
|
}
|
|
|
|
# adjust form type value:
|
|
if ($params{'formType'} =~ /^(pm)?id$/i) { # --type=id (or, previously: --type=pmid)
|
|
$params{'formType'} = "importID";
|
|
}
|
|
else { # --type=data
|
|
$params{'formType'} = "import";
|
|
}
|
|
|
|
# resolve any host shortcuts:
|
|
if (exists($hosts{$host})) {
|
|
$host = $hosts{$host};
|
|
}
|
|
elsif ($host !~ /^https?:\/\//i) {
|
|
$host = $hosts{'default'}; # can't resolve given host, reset back to default
|
|
}
|
|
|
|
# assign correct URL params based on the '-F|--format' option:
|
|
if (exists($outputParams{'format'})) {
|
|
$format = $outputParams{'format'};
|
|
if ($format =~ /^(html|rtf|pdf|latex|latex_bbl|markdown|ascii)$/i) {
|
|
$outputParams{'submit'} = "Cite";
|
|
}
|
|
if ($format =~ /^(html|rtf|pdf|latex|latex_bbl|markdown|ascii)$/i) {
|
|
$format =~ s/^latex_bbl$/LaTeX .bbl/i;
|
|
$outputParams{'citeType'} = $format;
|
|
}
|
|
elsif ($format =~ /^(ads|bibtex|endnote|isi|ris|atom|mods|oai_dc|odf|srw(_dc|_mods)?|word)$/i) {
|
|
$outputParams{'submit'} = "Export";
|
|
$outputParams{'exportType'} = "file";
|
|
|
|
if ($format =~ /^(ads|bibtex|endnote|isi|ris)$/i) {
|
|
$outputParams{'exportFormat'} = $format;
|
|
}
|
|
elsif ($format =~ /^(atom|mods|oai_dc|odf|srw(_dc|_mods)?|word)$/i) {
|
|
$outputParams{'exportFormat'} = $format . " xml";
|
|
}
|
|
}
|
|
else {
|
|
$outputParams{'citeType'} = "ascii";
|
|
}
|
|
|
|
delete($outputParams{'format'});
|
|
}
|
|
|
|
# construct URL:
|
|
# (uses URI::URL)
|
|
$importScript = "import_modify.php";
|
|
$importURL = url($host . $importScript);
|
|
|
|
# initialize new user agent:
|
|
# (uses LWP::UserAgent)
|
|
$userAgent = LWP::UserAgent->new;
|
|
|
|
# set user agent string:
|
|
$userAgent->agent("refbase_import/" . $version . " (http://cli.refbase.net/) ");
|
|
|
|
# set cookie jar object:
|
|
# LWP will collect cookies and respond to cookie requests via its cookie jar, thus
|
|
# enabling the user agent to fetch a PHP session ID from the refbase login response
|
|
# and automatically resend it upon next import request
|
|
$userAgent->cookie_jar({ file => $cookieJarFile, autosave => 1 });
|
|
|
|
# attempt to authenticate using the given login credentials:
|
|
if (($loginParams{'loginEmail'} ne '') && ($loginParams{'loginPassword'} ne '')) {
|
|
$loginSuccessful = &login(0); # call the 'login' subroutine
|
|
}
|
|
else {
|
|
$loginSuccessful = 0;
|
|
}
|
|
|
|
if (!$loginSuccessful) {
|
|
print "Login failed! You provided an incorrect email address or password.\n\n";
|
|
exit;
|
|
}
|
|
|
|
# send POST request:
|
|
# (uses HTTP::Request::Common & HTTP::Response)
|
|
if ($params{'formType'} =~ /^importID$/i) { # --type=id (or, previously: --type=pmid)
|
|
$request = POST $importURL, \%params;
|
|
}
|
|
else { # --type=data
|
|
$params{'uploadFile'} = \@sourceFile;
|
|
$request = POST $importURL, Content_Type => 'form-data', Content => \%params;
|
|
}
|
|
|
|
$response = $userAgent->request($request);
|
|
|
|
if ($response->is_error()) {
|
|
print STDERR $response->status_line, "\n";
|
|
}
|
|
else {
|
|
$location = $response->header('Location');
|
|
|
|
if ($location ne '') {
|
|
if ($location =~ /show.php/) {
|
|
# display imported records:
|
|
foreach $key (keys %outputParams) {
|
|
$location .= "&" . $key . "=" . $outputParams{$key};
|
|
}
|
|
if ($location =~ /&headerMsg=\D*(\d+)/i) {
|
|
$location .= "&showRows=" . $1;
|
|
}
|
|
}
|
|
|
|
# construct URL:
|
|
# (uses URI::URL)
|
|
$responseURL = url($host . $location);
|
|
|
|
# send GET request:
|
|
# (uses HTTP::Request::Common & HTTP::Response)
|
|
$request = GET $responseURL;
|
|
$response = $userAgent->request($request); # or use: $response = $userAgent->get($responseURL);
|
|
}
|
|
|
|
binmode STDOUT;
|
|
print $response->content();
|
|
}
|
|
|
|
# --------------------------------------------------------------------------------
|
|
|
|
# Login with login credentials given in '%loginParams':
|
|
sub login
|
|
{
|
|
local ($status) = @_;
|
|
|
|
# construct URL:
|
|
# (uses URI::URL)
|
|
$loginScript = "user_login.php";
|
|
$loginURL = url($host . $loginScript);
|
|
|
|
# send POST request:
|
|
# (uses HTTP::Request::Common & HTTP::Response)
|
|
$request = POST $loginURL, \%loginParams;
|
|
|
|
$response = $userAgent->request($request);
|
|
|
|
if ($response->is_error()) {
|
|
print STDERR $response->status_line, "\n";
|
|
exit $status;
|
|
}
|
|
else {
|
|
$location = $response->header('Location');
|
|
|
|
# upon successful login, refbase will redirect to 'index.php'
|
|
if ($location =~ /index.php/) {
|
|
return 1; # login successful
|
|
}
|
|
else {
|
|
return 0; # login NOT successful
|
|
}
|
|
}
|
|
}
|
|
|
|
# --------------------------------------------------------------------------------
|
|
|
|
# Print usage and exit:
|
|
sub usage
|
|
{
|
|
local ($status) = @_;
|
|
|
|
print "\nrefbase_import command line client, v" . $version . " by Matthias Steffens, http://cli.refbase.net/\n\n"
|
|
|
|
. "Usage: refbase_import [OPTIONS] [FILE]\n\n"
|
|
|
|
. "Notes: - Two import modes are supported:\n"
|
|
. " 1) '--type=data' requires an import FILE to be specified;\n"
|
|
. " for supported import formats, see: http://import.refbase.net/\n"
|
|
. " 2) '--type=id' requires the '-p, --pmid' option or one of '--arxiv|--doi|--openurl' with\n"
|
|
. " one or more whitespace-delimited PubMed IDs, arXiv IDs, DOIs or OpenURLs, respectively.\n"
|
|
. " - Options syntax: [OPTION]=[VALUE], e.g. '-p=16351846' or '--pmid=\"16351846 16783713\"'.\n"
|
|
. " - For each option, default values can be specified at the top of the script.\n"
|
|
. " Current defaults are given in parentheses.\n\n"
|
|
|
|
. "General Options: -h, --help - display this help text\n"
|
|
. " -v, --version - display version information\n"
|
|
. " -X, --examples - display usage examples\n\n"
|
|
|
|
. "Import Options: -b, --skipbad - skip records with unrecognized data format ('" . $params{'skipBadRecords'} . "')\n"
|
|
. " possible values: 0, 1\n"
|
|
. " -i, --import - import all or only some records ('" . $params{'importRecordsRadio'} . "')\n"
|
|
. " possible values: all, only\n"
|
|
. " -p, --pmid, - IDs of records to import ('" . $params{'sourceIDs'} . "')\n"
|
|
. " --arxiv, supported IDs: PubMed ID (PMID), arXiv ID, DOI, OpenURL\n"
|
|
. " --doi, \n"
|
|
. " --openurl \n"
|
|
. " -r, --records - positional numbers and/or ranges of records to import ('" . $params{'importRecords'} . "')\n"
|
|
. " requires the '--import=only' option\n"
|
|
. " -t, --type - import type ('" . $params{'formType'} . "')\n"
|
|
. " possible values: data, id\n\n"
|
|
|
|
. "Output Options: -C, --style - citation style ('" . $outputParams{'citeStyle'} . "')\n"
|
|
. " -F, --format - output format ('" . $outputParams{'format'} . "')\n"
|
|
. " possible values: html, rtf, pdf, latex, latex_bbl, markdown, ascii,\n"
|
|
. " ads, bibtex, endnote, isi, ris, atom, mods, oai_dc,\n"
|
|
. " odf, srw_dc, srw_mods, word\n"
|
|
. " -L, --showlinks - hide/display links column in html output ('" . $outputParams{'showLinks'} . "')\n"
|
|
. " possible values: 0, 1\n"
|
|
. " -O, --order - sort order of returned records ('" . $outputParams{'citeOrder'} . "')\n"
|
|
. " possible values: author, year, type, type-year, creation-date\n"
|
|
. " -V, --view - view type of html output ('" . $outputParams{'viewType'} . "')\n"
|
|
. " possible values: web, print, mobile\n\n"
|
|
|
|
. "Server Options: -H, --host - URL of the refbase database ('" . $host . "')\n"
|
|
. " defined shortcuts: " . join(', ', sort keys(%hosts)) . "\n"
|
|
. " -P, --password - password for given '-U, --user' account";
|
|
|
|
if ($loginParams{'loginPassword'} ne '') {
|
|
print "\n (a default pwd has been defined)\n";
|
|
}
|
|
else {
|
|
print " ('')\n";
|
|
}
|
|
|
|
print " -U, --user - login email address of an existing refbase user with\n"
|
|
. " import permissions ('" . $loginParams{'loginEmail'} . "')\n\n";
|
|
|
|
exit $status;
|
|
}
|
|
|
|
# --------------------------------------------------------------------------------
|
|
|
|
# Print version number and exit:
|
|
sub version
|
|
{
|
|
local ($status) = @_;
|
|
print "\nrefbase_import command line client, version " . $version
|
|
. "\ncheck for updates at http://cli.refbase.net/\n\n";
|
|
exit $status;
|
|
}
|
|
|
|
# --------------------------------------------------------------------------------
|
|
|
|
# Print examples and exit:
|
|
sub examples
|
|
{
|
|
local ($status) = @_;
|
|
print <<'END_EXAMPLES';
|
|
|
|
--------------------------------------------------------------------------------
|
|
REFBASE_IMPORT USAGE EXAMPLES:
|
|
--------------------------------------------------------------------------------
|
|
|
|
1) Import BibTeX records from file 'import.bib' using the defaults defined
|
|
within the refbase_import script:
|
|
|
|
refbase_import import.bib
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
2) Import all Endnote records given in file 'import.enw' into your default
|
|
refbase database:
|
|
|
|
refbase_import -t=data -i=all import.enw
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
3) Take RIS records from file 'import.ris' but import only the first three as
|
|
well as the fifth and the tenth record into your local refbase database:
|
|
|
|
refbase_import -H=local -t=data -i=only -r=1-3,5,10 import.ris
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
4) Import MODS XML records from file 'mods.xml' into the refbase demo database
|
|
using the defaults defined within the refbase_import script:
|
|
|
|
refbase_import -H=http://demo.refbase.net/ -U=user@refbase.net -P=user mods.xml
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
5) Fetch two records from PubMed.gov via their PMID (i.e. the unique PubMed
|
|
identifier, in this example, records with PMIDs 16351846 and 16783713) and
|
|
import them into your local refbase database:
|
|
|
|
refbase_import -H=local -t=id -p="16351846 16783713"
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
6) Fetch two records from CrossRef.org via their DOI (i.e. the unique Document
|
|
Object Identifier, in this example, records with DOIs 10.3354/meps251037 and
|
|
10.1103/PhysRev.47.777) and import them into your local refbase database:
|
|
|
|
refbase_import -H=local -t=id --doi="10.3354/meps251037 10.1103/PhysRev.47.777"
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
7) Fetch three records from arXiv.org via their arXiv ID (i.e. the unique arXiv
|
|
identifier, in this example, records with arXiv IDs astro-ph/0609768, 0806.1829
|
|
and 0802.0204v1) and import them into your local refbase database:
|
|
|
|
refbase_import -H=local -t=id --arxiv="astro-ph/0609768 0806.1829 0802.0204v1"
|
|
|
|
--------------------------------------------------------------------------------
|
|
|
|
END_EXAMPLES
|
|
exit $status;
|
|
}
|
|
|
|
__END__
|