:: ------------------------------------------------------------------- :: :: download.bat -- Maintain a small local gnuwin32 package archive :: within the cmd console of windows XP :: :: author : Mathias Michaelis :: date : Feb 19, 2009 :: version: 0.6.21 :: :: Copyright (c) 2009 by Mathias Michaelis, :: :: ------------------------------------------------------------------- :: :: Permission is granted to copy, distribute and/or modify this document :: under the terms of the GNU Free Documentation License, Version 1.2 :: or any later version published by the Free Software Foundation; :: with no Invariant Sections, no Front-Cover Texts, and no Back-Cover :: Texts. A copy of the license is included in the file LICENSE.TXT :: :: :: Acknowledgments :: --------------- :: :: See file README.TXT in this directory. Thanks to all who advised me, :: gave me some hints, sent me some patches or encouraged me to go on! :: @echo off setlocal ENABLEEXTENSIONS set LC_ALL=C :: :: Customization :: ------------- :: :: Select one or more of the following mirrors first! Please separate :: the mirrors by space, e.g. GNUWIN32_MIRROR="umn unc". :: :: aleron (US) :: belnet (BE) :: easynews (US) :: heanet (IE) :: internap (US) :: jaist (JP) :: keihanna (JP) :: kent (UK) :: mesh (DE) :: optusnet (AU) :: ovh (FR) :: puzzle (CH) :: switch (CH) :: umn (US) :: unc (US) :: voxel (US) :: if .%GNUWIN32_MIRROR%==. set GNUWIN32_MIRROR=voxel :: :: Maximal number of trials to download the gnuwin32 project site on :: sourceforge.net :: set /a GNUWIN32_MAX_TRIALS=4 :: :: All project sites must now always be checked due to changes in sourceforge.net :: In previous revisions of download.bat this was set with option -a :: set GNUWIN32_ALL_PROJECT_SITES=TRUE :: :: Process command line options. This can be a list of some above :: mentioned mirrors and/or the following: :: :: -v: be verbose. :: -n: only create a new getgnuwin32.lst file, no download. :: set GNUWIN32_CMDLINE_MIRROR= :command_line if .%1==. goto command_line_end if "%1"=="-v" goto set_verbose if "%1"=="/v" goto set_verbose if "%1"=="-n" goto set_no_packages if "%1"=="/n" goto set_no_packages if "%1"=="-d" goto set_direct_download if "%1"=="/d" goto set_direct_download if "%1"=="-h" goto show_help_text if "%1"=="/h" goto show_help_text if "%1"=="-?" goto show_help_text if "%1"=="/?" goto show_help_text if "%1"=="--help" goto show_help_text if "%1"=="--version" goto show_version_text set GNUWIN32_CMDLINE_MIRROR=%GNUWIN32_CMDLINE_MIRROR% %1 shift goto command_line :set_verbose set GNUWIN32_VERBOSE=TRUE shift goto command_line :set_no_packages set GNUWIN32_NO_PACKAGES=TRUE shift goto command_line :set_direct_download set GNUWIN32_DIRECT_DOWNLOAD=TRUE shift goto command_line :show_help_text echo = download.bat (version 0.6.21^) echo = Copyright (c^) 2009 by Mathias Michaelis, ^ echo = This is free software; see the source for copying conditions. There is NO echo = warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. echo = echo = usage: echo = download --version echo = download --help echo = download [-v] [-n] [-d] echo = echo = description: echo = download.bat, started without any command line options, will try to do two echo = things: echo = 1. To get an actual file list of all gnuwin32 projects and store it into echo = the file getgnuwin32.lst. If this file exists already and there is no echo = file named getgnuwin32.bak, the original getgnuwin32.lst is copied to echo = getgnuwin32.bak first. echo = 2. The newest versions of all files listed in getgnuwin32.lst are down- echo = loaded, except the files listed in include.txt, for which exactly the echo = therein specified versions are downloaded, and except the files listed echo = in exclude.txt. echo = The behaviour of download.bat can be modified by the following options: echo = -v : verbose mode. There are more lines printed on screen. echo = -n : no downloads. Only step 1 is executed, step 2 is skipped. echo = -d : direct download. Skip step 1. This is only possible if there is a echo = file list located in getgnuwin32.lst. echo = goto bye :show_version_text echo = download.bat (version 0.6.21^) echo = Copyright (c^) 2009 by Mathias Michaelis, ^ echo = This is free software; see the source for copying conditions. There is NO echo = warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. echo = goto bye :: :: Verify if command lines are meaningful :: :command_line_end if not defined GNUWIN32_DIRECT_DOWNLOAD goto set_gnuwin32_mirror if not exist getgnuwin32.lst goto no_getgnuwin32_file if not defined GNUWIN32_NO_PACKAGES goto set_gnuwin32_mirror echo = Error: -n and -d option are specified together. So I have nothing to do! echo = goto bye :no_getgnuwin32_file echo = Error: -d option was specified, but no getgnuwin32.lst file is found. Please echo = get this file from somewhere or don't use the -d option to automati- echo = cally try to download this file. echo = goto bye :: :: If command line options were specified, overwrite environment :: variables :: :set_gnuwin32_mirror if "%GNUWIN32_CMDLINE_MIRROR%"=="" goto environment set GNUWIN32_MIRROR=%GNUWIN32_CMDLINE_MIRROR:~1% set GNUWIN32_LOAD= set GNUWIN32_CMDLINE_MIRROR= :: :: After the mirrors are known, the download address of that mirrors :: can be constructed (if not already done): :: :environment if not .%GNUWIN32_LOAD%==. goto set_wgetrc set GNUWIN32_LOAD=http://%GNUWIN32_MIRROR:"=%.dl.sourceforge.net/sourceforge/gnuwin32/ set GNUWIN32_LOAD=%GNUWIN32_LOAD: =.dl.sourceforge.net/sourceforge/gnuwin32/ http://% :: :: Perhaps the user has specified her/his own .wgetrc file by means :: of the WGETRC environment variable. If not, then use the one in :: the local bin directory :: :set_wgetrc if ".%WGETRC%"=="." set WGETRC=%CD%\bin\wget.ini :: :: If we are in verbose mode, tell what we're going to do :: if "%GNUWIN32_VERBOSE%"=="" goto start_download_project_site echo = echo = URL of file list: echo = https://sourceforge.net/project/showfiles.php?group_id=23617 echo = echo = List of mirrors (high priority first): echo = %GNUWIN32_MIRROR% echo = echo = List of URLs (high priority first): echo %GNUWIN32_LOAD% >loadlist.txt bin\sed -T "s/ *$//;s/.*/= &/;s/[^ =] /\n= /g" loadlist.txt echo = del loadlist.txt :: :: First, get the file list form the official project site(s) on sourceforge.net :: :start_download_project_site if defined GNUWIN32_DIRECT_DOWNLOAD goto prepare_download_packages if not "%GNUWIN32_VERBOSE%"=="" echo = Downloading file list from sourceforge.net ... if exist project.html del project.html if exist normal.html del normal.html if exist normal.tmp del normal.tmp set /a GNUWIN32_TRIALS=0 :restart_download_project_site if /i %GNUWIN32_TRIALS% GEQ %GNUWIN32_MAX_TRIALS% goto stop_download_project_site set /a GNUWIN32_RATE="250 / (GNUWIN32_TRIALS %% 10 + 1)" set /a GNUWIN32_TRIALS="GNUWIN32_TRIALS + 1" set /a GNUWIN32_ODD_TRIAL="GNUWIN32_TRIALS & 1" if /i %GNUWIN32_TRIALS% GTR 1 echo = if /i %GNUWIN32_TRIALS% EQU 2 echo = Download project.html failed. 2nd trial ... if /i %GNUWIN32_TRIALS% EQU 3 echo = Download project.html failed. 3rd trial ... if /i %GNUWIN32_TRIALS% GTR 3 echo = Download project.html failed. %GNUWIN32_TRIALS%th trial ... if /i %GNUWIN32_TRIALS% GTR 1 echo = (Limit download rate to %GNUWIN32_RATE%kB) if /i %GNUWIN32_TRIALS% GTR 1 echo = if /i %GNUWIN32_ODD_TRIAL% EQU 1 ( bin\wget --ca-certificate=bin\cacert.pem --no-cache --limit-rate=%GNUWIN32_RATE%k -O project.html https://sourceforge.net/project/showfiles.php?group_id=23617 ) else ( bin\wget --no-cache --limit-rate=%GNUWIN32_RATE%k -O project.html http://sourceforge.net/project/showfiles.php?group_id=23617 ) :: :: This download very often fails. So check if we've got the whole :: file. :: if not exist project.html goto restart_download_project_site if defined GNUWIN32_ALL_PROJECT_SITES goto create_project_list bin\sed -T -n "/<\/html *>/Ip" project.html >project.tmp bin\test -s project.tmp if errorlevel 1 goto restart_download_project_site goto end_download_project_site :: :: If only a part of project.html could be received, the probability is :: very high that this part contains a list of all individual gnuwin32 :: projects along with the corresponding urls. So try to follow this :: links. :: :create_project_list bin\sed -T -n "s/>/>\n/g;s/normal.html :: SAVE THE LATEST RELEASES TABLE TO project.tmp bin\sed -T -n "/\bsummary[ \t]*=[ \t]*\d034Latest\sfile\slistings/I,/<\/table>/Ip" normal.html >project.tmp :: CHECK LAST LINE OF project.tmp FOR bin\sed -T -n "$p" project.tmp | bin\sed -T -n "/<\/table>/Ip" >normal.tmp bin\test -s normal.tmp :: IF NOT FOUND (ERRORLEVEL 1) RESTART DOWNLOAD if errorlevel 1 goto restart_download_project_site :: DELETE TEMP FILE normal.html normal.tmp del normal.html normal.tmp :: OUTPUT EVERY LINE BEFORE THE WORD Download (SHOULD BE PROJECT DOWNLOAD INFO LINKS) bin\sed -T -n "/^Download$/{g;1!p;};h" project.tmp >project.html :: SCAN PREVIOUS OUTPUT FOR PROJECT DOWNLOAD INFO LINKS AND OUTPUT TO project.tmp bin\sed -T -n "s/\&/\&/Ig;s//http:\/\/sourceforge.net\1/Ip" project.html >project.tmp bin\test -s project.tmp :: IF DOWNLOAD INFO LINKS NOT FOUND (ERRORLEVEL 1) RESTART DOWNLOAD if errorlevel 1 goto restart_download_project_site :: DELETE TEMP FILE project.html del project.html :: WGET PROJECT DOWNLOAD INFO LINKS FOR DOWNLOAD LINKS for /f "delims=" %%i in (project.tmp) do bin\wget --no-cache -t7 -O - %%i >>project.html goto end_download_project_site :: :: If the download of the whole project site failed, try to use only :: a part of it. This part probably allows to create a list of all :: gnuwin32 sub projects. :: :stop_download_project_site if defined GNUWIN32_ALL_PROJECT_SITES goto ultimate_stop_download_project_site echo = echo = Warning: Download failed %GNUWIN32_MAX_TRIALS% times. We try no to browse echo = all individual gnuwin32 project sites instead of the main project site. echo = set GNUWIN32_ALL_PROJECT_SITES=TRUE set /a GNUWIN32_MAX_TRIALS="GNUWIN32_MAX_TRIALS + 4" if not exist project.html goto restart_download_project_site goto create_project_list :: :: If the download failed many times, try to use an already present :: getgnuwin32.lst :: :ultimate_stop_download_project_site if exist project.tmp del project.tmp if exist project.html del project.html if defined GNUWIN32_NO_PACKAGES goto failed_create_filelist echo = echo = Warning: Couldn't download the gnuwin32 project site from sourceforge.net. We echo = try to use an already present file list now. Note that this list must reside echo = in a file named getgnuwin32.lst. echo = goto prepare_download_packages :failed_create_filelist echo = Error: Couldn't download the gnuwin32 project site from sourceforge.net. So echo = a new file list can't be created. Try to restart this program when the server echo = at sourceforge.net is less busy. echo = goto bye :: :: If the gnuwin32 project site could be downloaded, normalize it. Every :: tag has to be in its own lins. Empty lines are removed. :: :end_download_project_site del project.tmp if not exist project.html goto getgnuwin32_empty if not "%GNUWIN32_VERBOSE%"=="" echo = Analysing file list ... bin\sed -T -n "s/>/>\n/g;s/normal.html del project.html :: :: Convert the normalized HTML file in plain text. Take only the :: information that is relevant here: The package list. :: This procedure depends very much on the way the HTML file is :: programmed. That's why it's a bit risky. If this script is not :: up-to-date, this step may fail! :: :: OUTPUT PACKAGE TABLES TO normal.tmp bin\sed -T -n "//I,/<\/table>/Ip" normal.html > normal.tmp :: OUTPUT PACKAGE INFORMATION TO getgnuwin32.tmp bin\sed -T -n "s/.*http:\/\/downloads.sourceforge.net\/gnuwin32\/\(.*-\(bin\|doc\|lib\|dep\)\.zip\)?.*$/\1/p" normal.tmp >getgnuwin32.tmp del normal.html normal.tmp :: :: Before we continue, we have to check if getgnuwin32.tmp contains some :: information. If yes, old file lists must be removed first. :: bin\test -s getgnuwin32.tmp if not errorlevel 1 goto create_new_filelist goto getgnuwin32_empty :getgnuwin32_empty echo = echo = Warning: Couldn't completely download the gnuwin32 project site from echo = sourceforge.net, or the downloaded file has not the format that is echo = expected here -- maybe this script is out of date. We try now to use echo = an already existing file list. This file list must reside in a file echo = named getgnuwin32.lst. echo = if exist getgnuwin32.tmp del getgnuwin32.tmp goto prepare_download_packages :: :: The file getgnuwin32.lst is ultimately used at this place :: :prepare_download_packages if exist getgnuwin32.lst goto clean_old_lists echo = Error: getgnuwin32.lst doesn't exist. Possibly the above download of the echo = gnuwin32 project web site on sourceforge.net failed. You have now two echo = possibilities: echo = echo = 1. Try to restart download.bat later, if possible at a time where the echo = server at sourceforge.net is less busy. echo = echo = 2. Ask a friend at another place to run download.bat with the -n option. echo = This will create getgnuwin32.lst, the file you need to continue. If echo = you have it, run download.bat with the -d option, that will direct echo = download the desired packages and skip the download of the gnuwin32 echo = project site. echo = goto bye :: :: Create a new getgnuwin32.lst and backup the old one :: :create_new_filelist if exist getgnuwin32.lst if not exist getgnuwin32.bak ren getgnuwin32.lst getgnuwin32.bak if exist getgnuwin32.lst del getgnuwin32.lst ren getgnuwin32.tmp getgnuwin32.lst if defined GNUWIN32_NO_PACKAGES goto bye goto clean_old_lists :: :: Clean up old file lists etc. :: :clean_old_lists if exist filelist.txt del filelist.txt if exist packages.txt del packages.txt :: :: Extract the filelist from temporary file filelist.tmp :: :extract_filelist bin\sort getgnuwin32.lst >filelist.txt :: :: Assume: filelist.txt is already sorted by filenames and fileversion :: so that new versions are listed first. :: :: Now create a package list from the files with the newest versions :: :: bin\sed -T -n "/^[^ ]/s/\(.*\)-...\.zip/\1/p" filelist.txt >packages.tmp :: bin\sed -T ":x;N;s/^\(\([^-]*-[^[:digit:]]\)*[^-]*\)\(-[[:digit:]].*\)\?\n\1\(-[[:digit:]].*\)\?$/\1\3/;tx;P;D" packages.tmp >packages.txt :: :: New: getgnuwin32.lst contains also package information. This simplifies :: the process of generating a package list. :: bin\sed -T -n "s/^\(.*\)-...\.zip/\1/p" getgnuwin32.lst >packages.txt :: :: Cancel all packages which names are listed within the include.txt and :: the exclude.txt files. :: if exist include.txt bin\sed -T -n "s/\s*#.*$//;tx;:x;s/^\s*$//;t;s/\s*//g;p" include.txt >include.tmp if exist include.tmp bin\sed -T "s/^\(\([^-]*-[^[:digit:]]\)*[^-]*\).*$/\1 -/" include.tmp >exclude.tmp if exist exclude.txt bin\sed -T -n "s/\s*#.*$//;tx;:x;s/^\s*$//;t;s/^\s*\([^[:space:]]*\)\s*$/\1 -/p" exclude.txt >>exclude.tmp if not exist exclude.tmp goto add_include bin\sort -u packages.txt exclude.tmp -o packages.tmp bin\sed -T "/^.* -$/{:x;N;s/^\([^\n]*\) -\n\1-[[:digit:]].*$/\1 -/I;tx;D}" packages.tmp >packages.txt del packages.tmp del exclude.tmp :: :: Add packages in include.txt :: :add_include if not exist include.tmp goto shuffle copy include.tmp+packages.txt packages.tmp >nul del packages.txt del include.tmp ren packages.tmp packages.txt :: :: Shuffle package list and file list :: :shuffle bin\sed -T "s/^.*[^-]$/&-/" packages.txt >packages.tmp bin\sort filelist.txt packages.tmp -o filelist.tmp del packages.tmp :: :: Cancel all files which don't belong to a package :: bin\sed -T -n "h;N;s/^\([^\n]\+\)\n\1/\1/;tx;D;:x;G;P;D" filelist.tmp >filelist.txt del filelist.tmp :: :: Download the desired files from the server :: :download_packages if defined GNUWIN32_NO_PACKAGES goto bye if not "%GNUWIN32_VERBOSE%"=="" echo = Starting download of all files found within the file list ... if not exist packages goto makepackages if not exist oldpacks goto makeoldpacks move /y packages\*.* oldpacks >nul rmdir /s /q packages goto makepackages :makeoldpacks ren packages oldpacks :makepackages mkdir packages :download echo %GNUWIN32_LOAD% >loadlist.tmp bin\sed -T "s/ *$//;s/ /\n/g" loadlist.tmp >loadlist.txt del loadlist.tmp cd packages for /f %%f in (..\filelist.txt) do ( if not exist ..\oldpacks\%%f ( for /f %%m in (..\loadlist.txt) do ( if not exist %%f ..\bin\wget "%%m%%f" ) ) else ( move /y ..\oldpacks\%%f . ) ) cd .. del loadlist.txt :: :: Clean up things :: if not exist oldpacks goto bye dir /b oldpacks >oldpacks.txt bin\test -s oldpacks.txt if errorlevel 1 rmdir /s /q oldpacks if exist oldpacks.txt del oldpacks.txt :bye endlocal