Update OpenSSL to 1.0.1g and statically link OpenVPN with it

1 files changed, 410 insertions, 34 deletions

diff --git a/main/openssl/import_openssl.sh b/main/openssl/import_openssl.sh
index 6f601989..3f581530 100755
--- a/main/openssl/import_openssl.sh
+++ b/main/openssl/import_openssl.sh
@@ -27,6 +27,14 @@
 set -e
 trap  "echo WARNING: Exiting on non-zero subprocess exit code" ERR;
 
+# Make sure we're in the right directory.
+cd $(dirname $0)
+
+# Ensure consistent sorting order / tool output.
+export LANG=C
+export LC_ALL=C
+PERL_EXE="perl -C0"
+
 function die() {
   declare -r message=$1
 
@@ -56,7 +64,7 @@ function main() {
     die "openssl.version not found"
   fi
 
-  source openssl.version
+  source ./openssl.version
   if [ "$OPENSSL_VERSION" == "" ]; then
     die "Invalid openssl.version; see README.android for more information"
   fi
@@ -68,7 +76,7 @@ function main() {
     die "openssl.config not found"
   fi
 
-  source openssl.config
+  source ./openssl.config
   if [ "$CONFIGURE_ARGS" == "" -o "$UNNEEDED_SOURCES" == "" -o "$NEEDED_SOURCES" == "" ]; then
     die "Invalid openssl.config; see README.android for more information"
   fi
@@ -83,7 +91,7 @@ function main() {
     declare -r patch=$1
     shift || usage "No patch file specified."
     [ -d $OPENSSL_DIR ] || usage "$OPENSSL_DIR not found, did you mean to use generate?"
-    [ -d $OPENSSL_DIR_ORIG_ORIG ] || usage "$OPENSSL_DIR_ORIG not found, did you mean to use generate?"
+    [ -d $OPENSSL_DIR_ORIG ] || usage "$OPENSSL_DIR_ORIG not found, did you mean to use generate?"
     regenerate $patch
   elif [ "$command" = "generate" ]; then
     declare -r patch=$1
@@ -96,6 +104,317 @@ function main() {
   fi
 }
 
+# Compute the name of an assembly source file generated by one of the
+# gen_asm_xxxx() functions below. The logic is the following:
+# - if "$2" is not empty, output it directly
+# - otherwise, change the file extension of $1 from .pl to .S and output
+#   it.
+# Usage: default_asm_file "$1" "$2"
+#     or default_asm_file "$@"
+#
+# $1: generator path (perl script)
+# $2: optional output file name.
+function default_asm_file () {
+  if [ "$2" ]; then
+    echo "$2"
+  else
+    echo "${1%%.pl}.S"
+  fi
+}
+
+# Generate an ARM assembly file.
+# $1: generator (perl script)
+# $2: [optional] output file name
+function gen_asm_arm () {
+  local OUT
+  OUT=$(default_asm_file "$@")
+  $PERL_EXE "$1" > "$OUT"
+}
+
+function gen_asm_mips () {
+  local OUT
+  OUT=$(default_asm_file "$@")
+  # The perl scripts expect to run the target compiler as $CC to determine
+  # the endianess of the target. Setting CC to true is a hack that forces the scripts
+  # to generate little endian output
+  CC=true $PERL_EXE "$1" o32 > "$OUT"
+}
+
+function gen_asm_x86 () {
+  local OUT
+  OUT=$(default_asm_file "$@")
+  $PERL_EXE "$1" elf -fPIC > "$OUT"
+}
+
+function gen_asm_x86_64 () {
+  local OUT
+  OUT=$(default_asm_file "$@")
+  $PERL_EXE "$1" elf "$OUT" > "$OUT"
+}
+
+
+# Filter all items in a list that match a given pattern.
+# $1: space-separated list
+# $2: egrep pattern.
+# Out: items in $1 that match $2
+function filter_by_egrep() {
+  declare -r pattern=$1
+  shift
+  echo "$@" | tr ' ' '\n' | grep -e "$pattern" | tr '\n' ' '
+}
+
+# Sort and remove duplicates in a space-separated list
+# $1: space-separated list
+# Out: new space-separated list
+function uniq_sort () {
+  echo "$@" | tr ' ' '\n' | sort -u | tr '\n' ' '
+}
+
+function print_autogenerated_header() {
+  echo "# Auto-generated - DO NOT EDIT!"
+  echo "# To regenerate, edit openssl.config, then run:"
+  echo "#     ./import_openssl.sh import /path/to/openssl-$OPENSSL_VERSION.tar.gz"
+  echo "#"
+}
+
+# Run Configure and generate headers
+# $1: 32 for 32-bit arch, 64 for 64-bit arch, trusty for Trusty
+# $2: 1 if building for static version
+# Out: returns the cflags and depflags in variable $flags
+function generate_build_config_headers() {
+  chmod +x ./Configure
+  local configure_args_bits=CONFIGURE_ARGS_$1
+  local configure_args_stat=''
+  local outname=$1
+  if [ $2 -eq "1" ] ; then
+      configure_args_stat=CONFIGURE_ARGS_STATIC
+      outname="static-$1"
+  fi
+
+  if [ $1 == "trusty" ] ; then
+    PERL=/usr/bin/perl ./Configure $CONFIGURE_ARGS_TRUSTY
+  else
+    PERL=/usr/bin/perl ./Configure $CONFIGURE_ARGS ${!configure_args_bits} ${!configure_args_stat}
+  fi
+
+  rm -f apps/CA.pl.bak crypto/opensslconf.h.bak
+  mv -f crypto/opensslconf.h crypto/opensslconf-$outname.h
+  cp -f crypto/opensslconf-$outname.h include/openssl/opensslconf-$outname.h
+
+  local tmpfile=$(mktemp tmp.XXXXXXXXXX)
+  (grep -e -D Makefile | grep -v CONFIGURE_ARGS= | grep -v OPTIONS= | \
+      grep -v -e -DOPENSSL_NO_DEPRECATED) > $tmpfile
+  declare -r cflags=$(filter_by_egrep "^-D" $(grep -e "^CFLAG=" $tmpfile))
+  declare -r depflags=$(filter_by_egrep "^-D" $(grep -e "^DEPFLAG=" $tmpfile))
+  rm -f $tmpfile
+
+  flags="$cflags $depflags"
+}
+
+# Run Configure and generate makefiles
+function generate_build_config_mk() {
+  chmod +x ./Configure
+  for bits in 32 64 trusty; do
+    # Header flags are output in $flags, first static, then dynamic
+    generate_build_config_headers $bits 1
+    local flags_static=$flags
+    generate_build_config_headers $bits
+
+    echo "Generating build-config-$bits.mk"
+    (
+      print_autogenerated_header
+
+      echo "openssl_cflags_$bits := \\"
+      for flag in $flags ; do echo "  $flag \\" ; done
+      echo ""
+
+      echo "openssl_cflags_static_$bits := \\"
+      for flag in $flags_static; do echo "  $flag \\" ; done
+      echo ""
+    ) > ../build-config-$bits.mk
+  done
+}
+
+# Generate crypto/opensslconf.h file including arch-specific files
+function generate_opensslconf_h() {
+  echo "Generating opensslconf.h"
+  (
+  echo "// Auto-generated - DO NOT EDIT!"
+  echo "#ifndef OPENSSL_SYS_TRUSTY"
+  echo "#if defined(__LP64__)"
+  echo "#include \"opensslconf-64.h\""
+  echo "#else"
+  echo "#include \"opensslconf-32.h\""
+  echo "#endif"
+  echo "#else"
+  echo "#include \"opensslconf-trusty.h\""
+  echo "#endif"
+  ) > crypto/opensslconf.h
+  # Generate a compatible version for the static library builds
+  echo "Generating opensslconf-static.h"
+  (
+  echo "// Auto-generated - DO NOT EDIT!"
+  echo "#if defined(__LP64__)"
+  echo "#include \"opensslconf-static-64.h\""
+  echo "#else"
+  echo "#include \"opensslconf-static-32.h\""
+  echo "#endif"
+  ) > crypto/opensslconf-static.h
+  # move it to output include files as well
+  cp -f crypto/opensslconf-static.h include/openssl/opensslconf-static.h
+}
+
+# Return the value of a computed variable name.
+# E.g.:
+#   FOO=foo
+#   BAR=bar
+#   echo $(var_value FOO_$BAR)   -> prints the value of ${FOO_bar}
+# $1: Variable name
+# Out: variable value
+var_value() {
+  # Note: don't use 'echo' here, because it's sensitive to values
+  #       that begin with an underscore (e.g. "-n")
+  eval printf \"%s\\n\" \$$1
+}
+
+# Same as var_value, but returns sorted output without duplicates.
+# $1: Variable name
+# Out: variable value (if space-separated list, sorted with no duplicates)
+var_sorted_value() {
+  uniq_sort $(var_value $1)
+}
+
+# Print the definition of a given variable in a GNU Make build file.
+# $1: Variable name (e.g. common_src_files)
+# $2: prefix for each variable contents
+# $3+: Variable value (e.g. list of sources)
+print_vardef_with_prefix_in_mk() {
+  declare -r varname=$1
+  declare -r prefix=$2
+  shift
+  shift
+  if [ -z "$1" ]; then
+    echo "$varname :="
+  else
+    echo "$varname := \\"
+    for src; do
+      echo "  $prefix$src \\"
+    done
+  fi
+  echo ""
+}
+# Print the definition of a given variable in a GNU Make build file.
+# $1: Variable name (e.g. common_src_files)
+# $2+: Variable value (e.g. list of sources)
+print_vardef_in_mk() {
+  declare -r varname=$1
+  shift
+  print_vardef_with_prefix_in_mk $varname "" $@
+}
+
+# Same as print_vardef_in_mk, but print a CFLAGS definition from
+# a list of compiler defines.
+# $1: Variable name (e.g. common_cflags)
+# $2: List of defines (e.g. OPENSSL_NO_CAMELLIA ...)
+print_defines_in_mk() {
+  declare -r varname=$1
+  shift
+  if [ -z "$1" ]; then
+    echo "$varname :="
+  else
+    echo "$varname := \\"
+    for def; do
+    echo "  -D$def \\"
+    done
+  fi
+  echo ""
+}
+
+# Generate a configuration file like Crypto-config.mk
+# This uses variable definitions from openssl.config to build a config
+# file that can compute the list of target- and host-specific sources /
+# compiler flags for a given component.
+#
+# $1: Target file name.  (e.g. Crypto-config.mk)
+# $2: Variable prefix.   (e.g. CRYPTO)
+# $3: "host" or "target"
+function generate_config_mk() {
+  declare -r output="$1"
+  declare -r prefix="$2"
+  declare -r all_archs="arm arm64 x86 x86_64 mips"
+
+  echo "Generating $(basename $output)"
+  (
+    print_autogenerated_header
+    echo \
+"# This script will append to the following variables:
+#
+#    LOCAL_CFLAGS
+#    LOCAL_C_INCLUDES
+#    LOCAL_SRC_FILES_\$(TARGET_ARCH)
+#    LOCAL_SRC_FILES_\$(TARGET_2ND_ARCH)
+#    LOCAL_CFLAGS_\$(TARGET_ARCH)
+#    LOCAL_CFLAGS_\$(TARGET_2ND_ARCH)
+#    LOCAL_ADDITIONAL_DEPENDENCIES
+
+
+LOCAL_ADDITIONAL_DEPENDENCIES += \$(LOCAL_PATH)/$(basename $output)
+"
+
+    common_defines=$(var_sorted_value OPENSSL_${prefix}_DEFINES)
+    print_defines_in_mk common_cflags $common_defines
+
+    common_sources=$(var_sorted_value OPENSSL_${prefix}_SOURCES)
+    print_vardef_in_mk common_src_files $common_sources
+
+    common_includes=$(var_sorted_value OPENSSL_${prefix}_INCLUDES)
+    print_vardef_with_prefix_in_mk common_c_includes external/openssl/ $common_includes
+
+    for arch in $all_archs; do
+      arch_defines=$(var_sorted_value OPENSSL_${prefix}_DEFINES_${arch})
+      print_defines_in_mk ${arch}_cflags $arch_defines
+
+      arch_sources=$(var_sorted_value OPENSSL_${prefix}_SOURCES_${arch})
+      print_vardef_in_mk ${arch}_src_files $arch_sources
+
+      arch_exclude_sources=$(var_sorted_value OPENSSL_${prefix}_SOURCES_EXCLUDES_${arch})
+      print_vardef_in_mk ${arch}_exclude_files $arch_exclude_sources
+
+    done
+
+    if [ $3 == "target" ]; then
+      echo "
+LOCAL_CFLAGS += \$(common_cflags)
+LOCAL_C_INCLUDES += \$(common_c_includes)"
+      for arch in $all_archs; do
+        echo "
+LOCAL_SRC_FILES_${arch} += \$(filter-out \$(${arch}_exclude_files),\$(common_src_files) \$(${arch}_src_files))
+LOCAL_CFLAGS_${arch} += \$(${arch}_cflags)"
+      done
+    else
+      echo "
+ifeq (\$(HOST_OS)-\$(HOST_ARCH),linux-x86)
+ifneq (\$(BUILD_HOST_64bit),)
+host_arch := x86_64
+else
+host_arch := x86
+endif
+else
+ifeq (\$(HOST_OS)-\$(HOST_ARCH),linux-x86_64)
+host_arch := x86_64
+else
+\$(warning Unknown host architecture \$(HOST_OS)-\$(HOST_ARCH))
+host_arch := unknown
+endif
+endif
+
+LOCAL_CFLAGS     += \$(common_cflags) \$(\$(host_arch)_cflags)
+LOCAL_C_INCLUDES += \$(common_c_includes) \$(local_c_includes)
+LOCAL_SRC_FILES  += \$(filter-out \$(\$(host_arch)_exclude_files), \$(common_src_files) \$(\$(host_arch)_src_files))"
+    fi
+  ) > "$output"
+}
+
 function import() {
   declare -r OPENSSL_SOURCE=$1
 
@@ -104,18 +423,8 @@ function import() {
 
   cd $OPENSSL_DIR
 
-  # Configure source (and print Makefile defines for review, see README.android)
-  ./Configure $CONFIGURE_ARGS
-  rm -f apps/CA.pl.bak crypto/opensslconf.h.bak
-  echo
-  echo BEGIN Makefile defines to compare with android-config.mk
-  echo
-  grep -e -D Makefile | grep -v CONFIGURE_ARGS= | grep -v OPTIONS= | grep -v -e -DOPENSSL_NO_DEPRECATED
-  echo
-  echo END Makefile defines to compare with android-config.mk
-  echo
-
-  # TODO(): Fixup android-config.mk
+  generate_build_config_mk
+  generate_opensslconf_h
 
   cp -f LICENSE ../NOTICE
   touch ../MODULE_LICENSE_BSD_LIKE
@@ -129,17 +438,58 @@ function import() {
     fi
   done
 
-  # Copy Makefiles
-  cp ../patches/apps_Android.mk apps/Android.mk
-  cp ../patches/crypto_Android.mk crypto/Android.mk
-  cp ../patches/ssl_Android.mk ssl/Android.mk
-
-  # Generate asm
-  perl crypto/aes/asm/aes-armv4.pl         > crypto/aes/asm/aes-armv4.s
-  perl crypto/bn/asm/armv4-mont.pl         > crypto/bn/asm/armv4-mont.s
-  perl crypto/sha/asm/sha1-armv4-large.pl  > crypto/sha/asm/sha1-armv4-large.s
-  perl crypto/sha/asm/sha256-armv4.pl      > crypto/sha/asm/sha256-armv4.s
-  perl crypto/sha/asm/sha512-armv4.pl      > crypto/sha/asm/sha512-armv4.s
+  # Generate arm asm
+  gen_asm_arm crypto/aes/asm/aes-armv4.pl
+  gen_asm_arm crypto/bn/asm/armv4-gf2m.pl
+  gen_asm_arm crypto/bn/asm/armv4-mont.pl
+  gen_asm_arm crypto/modes/asm/ghash-armv4.pl
+  gen_asm_arm crypto/sha/asm/sha1-armv4-large.pl
+  gen_asm_arm crypto/sha/asm/sha256-armv4.pl
+  gen_asm_arm crypto/sha/asm/sha512-armv4.pl
+
+  # Generate mips asm
+  gen_asm_mips crypto/aes/asm/aes-mips.pl
+  gen_asm_mips crypto/bn/asm/mips.pl crypto/bn/asm/bn-mips.S
+  gen_asm_mips crypto/bn/asm/mips-mont.pl
+  gen_asm_mips crypto/sha/asm/sha1-mips.pl
+  gen_asm_mips crypto/sha/asm/sha512-mips.pl crypto/sha/asm/sha256-mips.S
+
+  # Generate x86 asm
+  gen_asm_x86 crypto/x86cpuid.pl
+  gen_asm_x86 crypto/aes/asm/aes-586.pl
+  gen_asm_x86 crypto/aes/asm/vpaes-x86.pl
+  gen_asm_x86 crypto/aes/asm/aesni-x86.pl
+  gen_asm_x86 crypto/bn/asm/bn-586.pl
+  gen_asm_x86 crypto/bn/asm/co-586.pl
+  gen_asm_x86 crypto/bn/asm/x86-mont.pl
+  gen_asm_x86 crypto/bn/asm/x86-gf2m.pl
+  gen_asm_x86 crypto/modes/asm/ghash-x86.pl
+  gen_asm_x86 crypto/sha/asm/sha1-586.pl
+  gen_asm_x86 crypto/sha/asm/sha256-586.pl
+  gen_asm_x86 crypto/sha/asm/sha512-586.pl
+  gen_asm_x86 crypto/md5/asm/md5-586.pl
+  gen_asm_x86 crypto/des/asm/des-586.pl
+  gen_asm_x86 crypto/des/asm/crypt586.pl
+  gen_asm_x86 crypto/bf/asm/bf-586.pl
+
+  # Generate x86_64 asm
+  gen_asm_x86_64 crypto/x86_64cpuid.pl
+  gen_asm_x86_64 crypto/sha/asm/sha1-x86_64.pl
+  gen_asm_x86_64 crypto/sha/asm/sha512-x86_64.pl crypto/sha/asm/sha256-x86_64.S
+  gen_asm_x86_64 crypto/sha/asm/sha512-x86_64.pl
+  gen_asm_x86_64 crypto/modes/asm/ghash-x86_64.pl
+  gen_asm_x86_64 crypto/aes/asm/aesni-x86_64.pl
+  gen_asm_x86_64 crypto/aes/asm/vpaes-x86_64.pl
+  gen_asm_x86_64 crypto/aes/asm/bsaes-x86_64.pl
+  gen_asm_x86_64 crypto/aes/asm/aes-x86_64.pl
+  gen_asm_x86_64 crypto/aes/asm/aesni-sha1-x86_64.pl
+  gen_asm_x86_64 crypto/md5/asm/md5-x86_64.pl
+  gen_asm_x86_64 crypto/bn/asm/modexp512-x86_64.pl
+  gen_asm_x86_64 crypto/bn/asm/x86_64-mont.pl
+  gen_asm_x86_64 crypto/bn/asm/x86_64-gf2m.pl
+  gen_asm_x86_64 crypto/bn/asm/x86_64-mont5.pl
+  gen_asm_x86_64 crypto/rc4/asm/rc4-x86_64.pl
+  gen_asm_x86_64 crypto/rc4/asm/rc4-md5-x86_64.pl
 
   # Setup android.testssl directory
   mkdir android.testssl
@@ -159,6 +509,14 @@ function import() {
 
   cd ..
 
+  generate_config_mk Crypto-config-target.mk CRYPTO target
+  generate_config_mk Crypto-config-host.mk CRYPTO host
+  generate_config_mk Crypto-config-trusty.mk CRYPTO_TRUSTY target
+  generate_config_mk Ssl-config-target.mk SSL target
+  generate_config_mk Ssl-config-host.mk SSL host
+  generate_config_mk Apps-config-target.mk APPS target
+  generate_config_mk Apps-config-host.mk APPS host
+
   # Prune unnecessary sources
   prune
 
@@ -196,6 +554,24 @@ function generate() {
   cleantar
 }
 
+# Find all files in a sub-directory that are encoded in ISO-8859
+# $1: Directory.
+# Out: list of files in $1 that are encoded as ISO-8859.
+function find_iso8859_files() {
+  find $1 -type f -print0 | xargs -0 file --mime-encoding | grep -i "iso-8859" | cut -d: -f1
+}
+
+# Convert all ISO-8859 files in a given subdirectory to UTF-8
+# $1: Directory name
+function convert_iso8859_to_utf8() {
+  declare -r iso_files=$(find_iso8859_files "$1")
+  for iso_file in $iso_files; do
+    iconv --from-code iso-8859-1 --to-code utf-8 $iso_file > $iso_file.tmp
+    rm -f $iso_file
+    mv $iso_file.tmp $iso_file
+  done
+}
+
 function untar() {
   declare -r OPENSSL_SOURCE=$1
   declare -r readonly=$2
@@ -205,11 +581,11 @@ function untar() {
 
   # Process new source
   tar -zxf $OPENSSL_SOURCE
-  mv $OPENSSL_DIR $OPENSSL_DIR_ORIG
+  convert_iso8859_to_utf8 $OPENSSL_DIR
+  cp -RfP $OPENSSL_DIR $OPENSSL_DIR_ORIG
   if [ ! -z $readonly ]; then
     find $OPENSSL_DIR_ORIG -type f -print0 | xargs -0 chmod a-w
   fi
-  tar -zxf $OPENSSL_SOURCE
 }
 
 function prune() {
@@ -241,7 +617,7 @@ function applypatches () {
   done
 
   # Cleanup patch output
-  find . -type f -name "*.orig" -print0 | xargs -0 rm -f
+  find . \( -type f -o -type l \) -name "*.orig" -print0 | xargs -0 rm -f
 
   cd ..
 }
@@ -250,12 +626,12 @@ function generatepatch() {
   declare -r patch=$1
 
   # Cleanup stray files before generating patch
-  find $BOUNCYCASTLE_DIR -type f -name "*.orig" -print0 | xargs -0 rm -f
-  find $BOUNCYCASTLE_DIR -type f -name "*~" -print0 | xargs -0 rm -f
+  find $OPENSSL_DIR -type f -name "*.orig" -print0 | xargs -0 rm -f
+  find $OPENSSL_DIR -type f -name "*~" -print0 | xargs -0 rm -f
+
+  # Find the files the patch touches and only keep those in the output patch
+  declare -r sources=`patch -p1 --dry-run -d $OPENSSL_DIR < $patch  | awk '/^patching file / { print $3 }'`
 
-  declare -r variable_name=OPENSSL_PATCHES_`basename $patch .patch | sed s/-/_/`_SOURCES
-  # http://tldp.org/LDP/abs/html/ivr.html
-  eval declare -r sources=\$$variable_name
   rm -f $patch
   touch $patch
   for i in $sources; do