Merged fix-927955-898665-pod2rst-issues & add a function in write-user-docs to fix the html that comes from sphinx, to really fix 898665

util/fix-pod2rst-output

@@ -1,68 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-my $file = shift @ARGV;
-die "I need a file" unless $file;
-die "$file is not a file" unless -f $file;
-open my $fh, '<', $file or die "Cannot open $file: $OS_ERROR";
-
-my $tool = shift @ARGV;
-die "I need a tool" unless $tool;
-
-print ".. program:: $tool\n\n",
-      ('=' x (length($tool) + 11)), "\n",
-      ":program:`$tool`\n",
-      ('=' x (length($tool) + 11)), "\n\n";
-
-$INPUT_RECORD_SEPARATOR = '';
-
-my $in_code_block = 0;
-my $section       = '';
-
-while (my $para = <$fh>) {
-   next if $para =~ m/^\.\. highlight:: perl/;
-
-   $in_code_block = $para =~ m/^\s{2,}/ ? 1 : 0;
-
-   if ($para =~ m/^\*{2,}\n([\w\s,-]+)\n\*{2,}$/m) {
-      print "$1\n",
-            ('=' x length $1),
-            "\n\n";
-      $section = $1;
-   }
-   elsif ($para =~ m/^Usage: /) {
-      $para =~ s/^Usage: //;
-      print "Usage\n",
-            "-----\n\n",
-            "::\n\n",
-            "  $para";
-   }
-   elsif ($para =~ m/^Examples:/) {
-      print "Examples\n",
-            "--------\n\n";
-   }
-   else {
-      $para =~ s/\.\. code-block:: perl/.. code-block:: bash/mg;
-      $para =~ s/`+$tool`+/$tool/g;
-      $para =~ s/([^\/])$tool/$1:program:`$tool`/g unless $in_code_block;
-      $para =~ s/^$tool/:program:`$tool`/gm;
-      $para =~ s/^--(\S+)$/.. option:: --$1/mg;
-      $para =~ s/"--(\S+)"/:option:`--$1`/g;
-      $para =~ s/\\\*/*/g;
-      $para =~ s/\\ //g;
-      $para =~ s/^[ ]+$//mg;
-      $para =~ s/^\n\n/\n/mg;
-      $para =~ s/code-block:: bash(\s+)CREATE/code-block:: sql$1CREATE/sg;
-      if ( ($section || '') eq 'OUTPUT' ) {
-         $para =~ s/^([A-Z_]+)\n\n/$1\n/;
-      }
-      print $para;
-   }
-}
-
-close $fh;
-exit;

util/pod2rst-fixed

@@ -0,0 +1,107 @@
+#!/usr/bin/env perl
+use 5.010; # for \K
+use strict;
+use warnings;
+use English qw(-no_match_vars);
+
+use IO::File;
+
+use HTML::Entities;
+
+use File::Basename qw(basename);
+use Pod::POM::View::Restructured;
+
+my $input_file = shift @ARGV or die "Need an input file";
+
+my $tool = basename($input_file);
+
+open my $in_fh, q{<:encoding(UTF-8)}, $input_file
+   or die "Cannot open $input_file: $!";
+
+open my $out_fh, q{>}, \my $out;
+
+my $conv   = Pod::POM::View::Restructured->new();
+my $output = $conv->convert_file($in_fh, undef, $out_fh, { link => \&format_links });
+
+close $in_fh;
+close $out_fh;
+
+if (!defined($output)) {
+   die "Failed to convert!";
+}
+
+my $header =
+      ".. program:: $tool\n\n" .
+      ('=' x (length($tool) + 11)) . "\n" .
+      ":program:`$tool`\n" .
+      ('=' x (length($tool) + 11)) . "\n\n";
+
+open my $in, q{<:encoding(UTF-8)}, \$out;
+
+local $INPUT_RECORD_SEPARATOR = '';
+
+my $in_code_block = 0;
+my $section       = '';
+
+my $fixed_output = '';
+
+while (my $para = <$in>) {
+   next if $para =~ m/^\.\. highlight:: perl/;
+
+   $in_code_block = $para =~ m/^\s{2,}/ ? 1 : 0;
+
+   if ($para =~ m/^\*{2,}\n([\w\s,-]+)\n\*{2,}$/m) {
+      $fixed_output .= "$1\n" .
+                       ('=' x length $1) .
+                       "\n\n";
+      $section = $1;
+   }
+   elsif ($para =~ m/^Usage: /) {
+      $para =~ s/^Usage: //;
+      $fixed_output .= "Usage\n"   .
+                       "-----\n\n" .
+                       "::\n\n"    .
+                       "  $para";
+   }
+   elsif ($para =~ m/^Examples:/) {
+      $fixed_output .= "Examples\n" .
+                       "--------\n\n";
+   }
+   else {
+      $para =~ s/\.\. code-block:: perl/.. code-block:: bash/mg;
+      $para =~ s/`+$tool`+/$tool/g;
+      $para =~ s/([^\/])$tool/$1:program:`$tool`/g unless $in_code_block;
+      $para =~ s/^$tool/:program:`$tool`/gm;
+      $para =~ s/^--(\S+)$/.. option:: --$1/mg;
+      $para =~ s/"--(\S+)"/:option:`--$1`/g;
+      $para =~ s/\\\*/*/g;
+      $para =~ s/\\ //g;
+      $para =~ s/^[ ]+$//mg;
+      $para =~ s/^\n\n/\n/mg;
+      $para =~ s/code-block:: bash(\s+)CREATE/code-block:: sql$1CREATE/sg;
+      $para =~ s/\Q.. option:: \E\K(.+)/test($1)/eg;
+      if ( ($section || '') eq 'OUTPUT' ) {
+         $para =~ s/^([A-Z_]+)\n\n/$1\n/;
+      }
+      $fixed_output .= $para;
+   }
+}
+
+sub test {
+   my ($t) = @_;
+   return $t =~ s/([\[\]])/\\$1/rg;
+}
+
+close $in;
+ 
+print $header . $fixed_output;
+
+sub format_links {
+   if ( my ($label, $url) = split /\|/, $_[0] ) {
+      return $url, $label;
+   }
+   else {
+      local $conv->{callbacks}{link};
+      return $conv->view_seq_link(@_)
+   }
+}

util/write-user-docs

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+#set -x
+
 # Usage: write-user-docs [TOOLS]
 #
 # This script writes/updates the user documentation.  User docs come from
@@ -64,24 +66,37 @@ RST_DIR=$DOCS_DIR/user
 # Subroutines
 # ############################################################################
 
+fix_html () {
+   local name="$1"
+   perl -MFile::Basename=basename -MFile::Slurp=read_file,write_file -le '
+      my $f    = shift;
+      my $tool = basename($f);
+      $tool    =~ s/\.html//;
+      my $out = read_file($f);
+      $out =~ s{
+\Q<dt id="\E(cmdoption-$tool--)\Q">\E\s*
+\Q<tt class="descname">--</tt><tt class="descclassname">\E([^<]+)
+\Q</tt><a class="headerlink" href="\E[^"]+"
+}{<dt id="$1$2">
+<tt class="descname">--$2</tt><tt class="descclassname"></tt><a class="headerlink" href="#$1$2"}xg;
+      write_file($f, $out);
+   ' $RST_DIR/html/$name.html
+}
+
 write_rst() {
    local file="$1"
+   local tool="$(basename $1)"
    if [ ! -f $file ]; then
       warn "$file does not exist"
       return
    fi
-   local tool=$(basename $file)
 
-   cat $file | pod2rst > $RST_DIR/$tool.orig.rst
+   $BRANCH/util/pod2rst-fixed $file > $RST_DIR/$tool.rst
    if [ $? -eq 0 ]; then
-      $BRANCH/util/fix-pod2rst-output $RST_DIR/$tool.orig.rst $tool \
-         > $RST_DIR/$tool.rst
       echo "Wrote $RST_DIR/$tool.rst"
    else
-      warn "Error writing $RST_DIR/tool.rst"
+      die "Error writing $RST_DIR/$tool.rst"
    fi
-
-   rm $RST_DIR/$tool.orig.rst
 }
 
 # Parse the head1 sections from percona-toolkit.pod and write them as
@@ -150,4 +165,17 @@ if [ $BUILD -eq 1 ]; then
    exit_status=$(( exit_status | $? ))
 fi
 
+if [ $# -gt 0 ]; then
+   for tool; do
+      name="$(basename $tool)"
+      fix_html $name
+   done
+else
+   for tool in `ls $BRANCH/bin/*`; do
+      name="$(basename $tool)"
+      fix_html $name
+   done
+fi
+
+
 exit $exit_status