#!/bin/sh -
#
# Copyright (C) 2007-2009  Internet Systems Consortium, Inc. ("ISC")
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

# $Id$

# Generate bibliographic references database for xml2rfc, etc by
# fetching data from the IETF Secretariat and the RFC Editor and
# whacking it into the formats we need.

main() {
    setup
    $rsync -tzq4	rsync://rsync.ietf.org/internet-drafts/1id-index.txt		xml/
    $rsync -tzq4	rsync://rsync.ietf.org/internet-drafts/all_id.txt		xml/
    $rsync -tzq4	rsync://ftp.rfc-editor.org/rfcs-text-only/rfc-index.xml		xml/
    $rsync -tzq4	rsync://ftp.rfc-editor.org/rfcs-text-only/rfc-index.xsd		xml/
    $wget -q -m -nd -4	http://greenbytes.de/tech/webdav/rfc2629xslt.zip		-P xml
    $wget -q -m -nd -4	http://www.w3.org/2002/01/tr-automation/tr.rdf			-P xml
    gen_rfc_refs	xml/rfc								xml/rfc-index.xml \
											xml/rfc-index.xsd
    gen_id_refs		xml/draft							xml/1id-index.txt
    gen_id_status	xml/ietf-id-status.xml						xml/all_id.txt
    gen_bibtex		xml/rfc.bib							xml/rfc-index.xml
    $unzip -a -q -f -o	xml/rfc2629xslt.zip						-d xml
    $trang		xml/rfc2629xslt/rfc2629.rnc					xml/rfc2629xslt/rfc2629.rng
    $chmod a-x		xml/rfc2629xslt/*
    $svn update		--quiet
    $svn add		--force								xml
    $svn ci		--message 'Reference database update'				xml
}

setup() {
  chmod=/bin/chmod
  iconv=/usr/local/bin/iconv
  mkdir=/bin/mkdir
  perl=/usr/local/bin/perl
  rm=/bin/rm
  rmdir=/bin/rmdir
  rsync=/usr/local/bin/rsync
  svn=/usr/local/bin/svn
  trang=/usr/local/bin/trang
  unzip=/usr/local/bin/unzip
  wget=/usr/local/bin/wget
  xmllint=/usr/local/bin/xmllint
  xsltproc=/usr/local/bin/xsltproc
}

# Fetch reference files from xml2rfc project at xml.resource.org.
# Sadly, this is very slow -- the code below is fine, the problem is
# that the network protocol it's using (checking the date of each file
# individually) is very slow (although not as slow as downloading
# without checking would be!).  Mirror or rsync would be much better,
# but neither will work with xml.resource.org.  Sigh.
#
# The code below uses xsltproc to fetch and parse the index page for
# the directory, then spits out a list of filenames for wget to check,
# one file at a time.  As screen scraping goes, this is very reliable:
# xsltproc has a real HTML parser, and the XSLT code below is telling
# xsltproc to extract and print the href attribute of every anchor
# element that matches a certain filename pattern.

xml2rfc_ref_fetch() {
  xsltproc <<'EOF' --html - $2 | wget -nv -nd -N -B $2 -P $1 -i -
    <xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
		   version="1.0">
      <xsl:output method="text" encoding="US-ASCII"/>
      <xsl:template match="//a[contains(@href, 'reference.')]">
	<xsl:value-of select="concat(@href, '&#10;')"/>
      </xsl:template>
      <xsl:template match="text()"/>
    </xsl:transform>
EOF
}

# Convert RFC Editor's rfc-index.xml into rfc2629 references suitable
# for use with xml2rfc et al.  Since the database is already XML, this
# is not a kludge, just normal XML processing.
#
# 95% of the work here was done by Bill Fenner (thanks!).
#
# Perl postprocessor is just to avoid updating files that haven't
# changed, so that makefile dependencies will work correctly.
#
# This hack could instead have been implemented as a Perl script using
# the XML::LibXSLT, but that would require the libxslt package anyway,
# so we'd already have xsltproc available, and XML::LibXSLT has
# occasionally give me reason to suspect that it has a memory leak
# that could interfere with our ability to process a large XML file.
#
# Occasionally the RFC Editor does something weird that breaks the
# XML.  Most such errors show up as violations of the corresponding
# XML schema, so before we do anything else we make sure that the
# current XML validates against the current schema.
#
# Well, it's 2019, this code is pretty antique, and it seems that
# Julian's RelaxNG schema hasn't caught up to what Heather is
# shipping, so the schema check has turned into nightly whining.
# Disable for now, perhaps forever.

gen_rfc_refs() {
# $xmllint --noout -schema $3 $2 || return
  $rm -rf $1.new
  test -d $1 || $mkdir $1 || return
  $xsltproc <<'EOF' --stringparam dir $1 - $2 | $perl -e '
    # Perl code starts here
    sub snarf {
      local $/;
      if (open(F, $_[0])) {
        my $x = <F>;
	close(F);
        return $x;
      } else {
        return undef;
      }
    }
    while (<>) {
      chomp;
      my $new_name = $_;
      s=[.]new(/[^/]+)$=$1=;
      my $old_name = $_;
      my $old_text = snarf($old_name);
      my $new_text = snarf($new_name);
      if ($new_text && $old_text ne $new_text) {
        rename($new_name, $old_name);
      } else {
        unlink($new_name);
      }
    }
    # Perl code ends here
  '
    <!-- XSLT code starts here -->

    <xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
		   xmlns:rfc="http://www.rfc-editor.org/rfc-index"
		   xmlns:exsl="http://exslt.org/common"
		   extension-element-prefixes="exsl"
		   exclude-result-prefixes="rfc"
		   version="1.0">

      <xsl:output method="text" encoding="US-ASCII"/>

      <xsl:param name="dir"/>

      <xsl:template match="rfc:rfc-entry">
	<xsl:variable name="file">
	  <xsl:value-of select="$dir"/>
	  <xsl:text>.new/reference.</xsl:text>
	  <xsl:choose>
	    <xsl:when test="starts-with(rfc:doc-id, 'RFC')">
	      <xsl:text>RFC.</xsl:text>
	      <xsl:value-of select="substring-after(rfc:doc-id, 'RFC')"/>
	    </xsl:when>
	    <xsl:otherwise>
	      <xsl:value-of select="rfc:doc-id"/>
	    </xsl:otherwise>
	  </xsl:choose>
	  <xsl:text>.xml</xsl:text>
	</xsl:variable>
	<xsl:value-of select="concat($file, '&#10;')"/>
	<exsl:document href="{$file}" method="xml" indent="yes" encoding="US-ASCII">
	  <reference>
	    <xsl:attribute name="anchor">
	      <xsl:value-of select="rfc:doc-id"/>
	    </xsl:attribute>
	    <front>
	      <title>
		<xsl:value-of select="rfc:title"/>
	      </title>
	      <xsl:if test="not(rfc:author[1])">
		<xsl:message>
		  <xsl:text>Eeek!  </xsl:text>
		  <xsl:value-of select="rfc:doc-id"/>
		  <xsl:text> has no authors listed!</xsl:text>
		</xsl:message>
	      </xsl:if>
	      <xsl:apply-templates select="rfc:author"/>
	      <date>
		<xsl:if test="rfc:date/rfc:month">
		  <xsl:attribute name="month">
		    <xsl:value-of select="rfc:date/rfc:month"/>
		  </xsl:attribute>
		</xsl:if>
		<xsl:attribute name="year">
		  <xsl:value-of select="rfc:date/rfc:year"/>
		</xsl:attribute>
	      </date>
	      <xsl:for-each select="rfc:keywords/rfc:kw">
		<keyword>
		  <xsl:value-of select="."/>
		</keyword>
	      </xsl:for-each>
	      <xsl:if test="rfc:abstract">
		<abstract>
		  <xsl:call-template name="parseabstract">
		    <xsl:with-param name="abstract" select="rfc:abstract"/>
		  </xsl:call-template>
		</abstract>
	      </xsl:if>
	    </front>
	    <xsl:variable name="rfcnum" select="number(substring-after(rfc:doc-id, 'RFC'))"/>
	    <seriesInfo name='RFC'>
	      <xsl:attribute name="value">
		<xsl:value-of select="$rfcnum"/>
	      </xsl:attribute>
	    </seriesInfo>
	    <xsl:for-each select="rfc:is-also/rfc:doc-id">
	      <!-- todo: better extraction of series and value -->
	      <seriesInfo>
		<xsl:attribute name="name">
		  <xsl:value-of select="substring(.,1,3)"/>
		</xsl:attribute>
		<xsl:attribute name="value">
		  <xsl:value-of select="number(substring(.,4))"/>
		</xsl:attribute>
	      </seriesInfo>
	    </xsl:for-each>
	    <xsl:for-each select="rfc:format">
	      <xsl:choose>
		<xsl:when test="rfc:file-format = 'PDF'">
		  <format type='PDF'>
		    <xsl:if test="rfc:char-count">
		      <xsl:attribute name="octets">
			<xsl:value-of select="rfc:char-count"/>
		      </xsl:attribute>
		    </xsl:if>
		    <xsl:attribute name="target">
		      <xsl:text>http://www.rfc-editor.org/rfc/rfc</xsl:text>
		      <xsl:value-of select="$rfcnum"/>
		      <xsl:text>.pdf</xsl:text>
		    </xsl:attribute>
		  </format>
		</xsl:when>
		<xsl:when test="rfc:file-format = 'ASCII'">
		  <format type='TXT'>
		    <xsl:if test="rfc:char-count">
		      <xsl:attribute name="octets">
			<xsl:value-of select="rfc:char-count"/>
		      </xsl:attribute>
		    </xsl:if>
		    <xsl:attribute name="target">
		      <xsl:text>http://www.rfc-editor.org/rfc/rfc</xsl:text>
		      <xsl:value-of select="$rfcnum"/>
		      <xsl:text>.txt</xsl:text>
		    </xsl:attribute>
		  </format>
		</xsl:when>
	      </xsl:choose>
	    </xsl:for-each>
	    <xsl:apply-templates mode="comment"/>
	  </reference>
	</exsl:document>
      </xsl:template>

      <xsl:template match="rfc:author">
	<xsl:variable name="fullname" select="normalize-space(rfc:name)"/>
	<xsl:variable name="initials" select="substring-before($fullname, ' ')"/>
	<xsl:variable name="surname" select="substring-after($fullname, ' ')"/>
	<xsl:if test="not($fullname)">
	  <xsl:message>
	    <xsl:text>Eeek!  </xsl:text>
	    <xsl:value-of select="../rfc:doc-id"/>
	    <xsl:text> has a nameless author!</xsl:text>
	  </xsl:message>
	</xsl:if>
	<author fullname="{$fullname}">
	  <xsl:if test="$initials">
	    <xsl:attribute name="initials">
	      <xsl:value-of select="$initials"/>
	    </xsl:attribute>
	  </xsl:if>
	  <xsl:if test="$surname">
	    <xsl:attribute name="surname">
	      <xsl:value-of select="$surname"/>
	    </xsl:attribute>
	  </xsl:if>
	  <organization/>
	</author>
      </xsl:template>

      <xsl:template name="parseabstract">
	<xsl:param name="abstract"/>
	<xsl:variable name="delimiter">&lt;br&gt;</xsl:variable>
	<xsl:choose>
	  <xsl:when test="contains($abstract, $delimiter)">
	    <xsl:call-template name="parseabstract">
	      <xsl:with-param name="abstract" select="substring-before($abstract, $delimiter)"/>
	    </xsl:call-template>
	    <xsl:call-template name="parseabstract">
	      <xsl:with-param name="abstract" select="substring-after($abstract, $delimiter)"/>
	    </xsl:call-template>
	  </xsl:when>
	  <xsl:otherwise>
	    <t>
	      <xsl:value-of select="$abstract"/>
	    </t>
	  </xsl:otherwise>
	</xsl:choose>
      </xsl:template>

      <xsl:template match="rfc:current-status | rfc:publication-status |
			   rfc:obsoleted-by   | rfc:obsoletes          |
			   rfc:updated-by     | rfc:updates            |
			   rfc:see-also       | rfc:notes"
		    mode="comment">
	<xsl:comment>
	  <xsl:value-of select="concat(' ', local-name(), ' ',
					    normalize-space(.), ' ')"/>
	</xsl:comment>
      </xsl:template>

      <xsl:template match="text()"/>

      <xsl:template match="text()" mode="comment"/>

    </xsl:transform>

    <!-- XSLT code ends here -->
EOF
  $rmdir $1.new
}

# Generate ietf-id-status database used by Julian Reschke's handy
# little check-ietf-references.xslt script.  Input is a tab-delimited
# file produced by the IETF Secretariat.  Parsing is totally ad hoc.

gen_id_status() {
  $perl -ne <$2 >$1 '
    BEGIN { print("<ietf-id-status>\n") }
    END   { print("</ietf-id-status>\n") }
    chomp;
    s/&/&amp;/g;
    s/</\&lt;/g;
    s/>/\&gt;/g;
    s/[""]/&quot;/g;
    next unless /^draft-/;
    ($name, $date, $status, $num) = split(/\t/);
    print("  <id name=\"$name\"\n",
	  "      date=\"$date\"\n",
	  "      status=\"$status\"");
    print("\n",
	  "      num=\"$num\"")
        if ($num);
    print("/>\n");
  '
}

# Generate a BibTeX database from the RFC index.  Semantics of this
# translation courtesy of Eric Rescorla, but this XSLT code runs a lot
# faster than Eric's original Perl implementation.

gen_bibtex() {
  $xsltproc <<'EOF' -o $1 - $2

    <xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"
		   xmlns:rfc="http://www.rfc-editor.org/rfc-index">

      <xsl:output method="text" encoding="US-ASCII"/>

      <xsl:template match="/">
	<xsl:text>@comment{Automatically generated, do not edit}&#10;</xsl:text>
	<xsl:for-each select="//rfc:rfc-entry">
	  <xsl:variable name="rfcnum" select="number(substring-after(rfc:doc-id, 'RFC'))"/>
	  <xsl:variable name="authors">
	    <xsl:value-of select="rfc:author[1]/rfc:name"/>
	    <xsl:for-each select="rfc:author[position() &gt; 1]/rfc:name">
	      <xsl:value-of select="concat(' and ', .)"/>
	    </xsl:for-each>
	  </xsl:variable>
	  <xsl:value-of select="
	    concat('&#10;',
		   '@misc{',                 rfc:doc-id,                 ',&#10;',
		   '  author       = {',     normalize-space($authors), '},&#10;',
		   '  title        = {{',    rfc:title,                '}},&#10;',
		   '  howpublished = {RFC ', $rfcnum,                   '},&#10;',
		   '  month        = {',     rfc:date/rfc:month,        '},&#10;',
		   '  year         = {',     rfc:date/rfc:year,         '}&#10;',
		   '}&#10;')"/>
	</xsl:for-each>
      </xsl:template>

    </xsl:transform>
EOF
}

# Generate references for Internet-Drafts by scraping another of the
# files generated by the IETF Secretariat.  We don't have enough
# information to fill in all the fields correctly, but this is
# probably close enough for most purposes.  Parsing is totally ad hoc.

gen_id_refs() {
  test -d $1 || $mkdir $1 || return
  $iconv -c -f UTF-8 -t US-ASCII $2 |
  $perl -e '
    use strict;

    my $dir = shift(@ARGV);
    die unless ($dir && -d $dir);

    my @months = qw(January	February	March
		    April	May		June
		    July	August		September
		    October	November	December);

    my $baseuri = "http://www.ietf.org/internet-drafts";

    my $text;
    my $wg_abbrev;
    my $wg_name;

    while (<>) {
	chomp;
	last if (/^\s*$/);
    }

    while (<>) {
	chomp;
	last if (/^\s*$/);
    }

    while (<>) {
	chomp;
	if (/^-+\s*$/ || /^\w.+ \(none\)$/) {
	    undef $wg_name;
	    undef $wg_abbrev;
	    undef $text;
	} elsif (/^(\w.+) \((\w+)\)$/) {
	    $wg_name = $1;
	    $wg_abbrev = $2;
	    undef $text;
	} elsif (/^.+$/) {
	    $text .= $_;
	} elsif ($text) {
	    $_ = $text;
	    undef $text;
	    s/^\s+//;
	    s/\s+$//;
	    s/\s+/ /g;
	    s/&/&amp;/g;
	    s/</\&lt;/g;
	    s/>/\&gt;/g;
	    my ($title, $authors, $year, $month, $day, $anchor, $number) = m{
		^"(.+)",\s
		([^""]*),\s
		(\d{4})-(\d{1,2})-(\d{1,2}),\s
		&lt;draft-([^""]+)-(\d{2,})\.\w+(?:,\.\w+)*&gt;$
	    }x;
	    if (!defined($number)) {
		warn("could not parse: $_");
		next;
	    }
	    $title =~ s/[""]/&quot;/g;
	    my @authors = split(/, /, $authors);
	    my %initials;
	    my %surname;
	    for (@authors) {
                $_ = "Sean P. Turner" if $_ eq "spt";
		my @w = split(/ /);
		$surname{$_} = pop(@w);
		$initials{$_} = join("", map {substr($_, 0, 1) . "."} @w);
	    }
	    $month = $months[$month - 1];
	    my $xml;
	    $xml .=
		qq(<?xml version="1.0" encoding="US-ASCII"?>\n) .
		qq(<!-- Automatically generated, do not edit. -->\n) .
		qq(\n) .
		qq(<reference anchor="I-D.$anchor">\n) .
		qq(  <front>\n) .
		qq(    <title>$title</title>\n);
	    $xml .=
		qq(    <author initials="$initials{$_}" surname="$surname{$_}" fullname="$_"><organization/></author>\n)
		foreach (@authors);
	    $xml .=
		qq(    <date day="$day" month="$month" year="$year"/>\n);
	    $xml .=
		qq(    <workgroup>$wg_name</workgroup>\n)
		if ($wg_name);
	    $xml .=
		qq(  </front>\n) .
		qq(  <seriesInfo name="Internet-Draft" value="draft-$anchor-$number"/>\n) .
		qq(  <format type="TXT" target="$baseuri/draft-$anchor-$number.txt"/>\n) .
		qq(</reference>\n);
	    my $file = "$dir/reference.I-D.$anchor.xml";
	    my $old_xml;
	    if (open(F, $file)) {
		local $/;
		$old_xml = <F>;
		close(F);
	    }
	    if ($xml ne $old_xml and (!open(F, ">$file.new") or
				      !print(F $xml) or
				      !close(F) or
				      !rename("$file.new", $file))) {
		warn("Could not write $file: $!");
		unlink("$file.new");
	    }
	}
    }
  ' $1
}

# Finally run something, now that we've defined all these functions.

main