[booh] / bin / booh-backend

#! /usr/bin/ruby
#
#                         *  BOOH  *
#
# A.k.a 'Best web-album Of the world, Or your money back, Humerus'.
#
# The acronyn sucks, however this is a tribute to Dragon Ball by
# Akira Toriyama, where the last enemy beaten by heroes of Dragon
# Ball is named "Boo". But there was already a free software project
# called Boo, so this one will be it "Booh". Or whatever.
#
#
# Copyright (c) 2005 Guillaume Cottenceau <gc3 at bluewin.ch>
#
# This software may be freely redistributed under the terms of the GNU
# public license version 2.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

require 'getoptlong'
require 'gettext'
include GetText
require 'rexml/document'
include REXML

require 'booh/booh-lib'
require 'booh/html-merges'

#- bind text domain as soon as possible because some _() functions are called early to build data structures
bindtextdomain("booh")

#- options
$options = [
    [ '--help',          '-h', GetoptLong::NO_ARGUMENT,       _("Get help message") ],
    [ '--version',       '-V', GetoptLong::NO_ARGUMENT,       _("Print version and exit") ],

    [ '--source',        '-s', GetoptLong::REQUIRED_ARGUMENT, _("Directory which contains original images/videos as files or subdirs") ],
    [ '--destination',   '-d', GetoptLong::REQUIRED_ARGUMENT, _("Directory which will contain the web-album") ],
#    [ '--clean',         '-c', GetoptLong::NO_ARGUMENT,       _("Clean destination directory") ],

    [ '--theme',         '-t', GetoptLong::REQUIRED_ARGUMENT, _("Select HTML theme to use") ],
    [ '--config',        '-C', GetoptLong::REQUIRED_ARGUMENT, _("File containing config listing images and videos within directories with captions") ],
    [ '--config-skel',   '-k', GetoptLong::REQUIRED_ARGUMENT, _("Filename where the script will output a config skeleton") ],
    [ '--merge-config',  '-M', GetoptLong::REQUIRED_ARGUMENT, _("File containing config listing, where to merge new/removed images/videos from --source, and change theme info") ],
    [ '--merge-config-onedir',  '-O', GetoptLong::REQUIRED_ARGUMENT, _("File containing config listing, for merging the subdir specified with --dir") ],
    [ '--merge-config-subdirs', '-U', GetoptLong::REQUIRED_ARGUMENT, _("File containing config listing, for merging the new subdirs down the subdir specified with --dir") ],
    [ '--dir',           '-D', GetoptLong::REQUIRED_ARGUMENT, _("Directory for merge with --merge-config-onedir or --merge-config-subdirs") ],
    [ '--use-config',    '-u', GetoptLong::REQUIRED_ARGUMENT, _("File containing config listing, where to change theme info") ],
    [ '--force',         '-f', GetoptLong::NO_ARGUMENT,       _("Force generation of album even if the GUI marked some directories as already generated") ],

    [ '--sizes',         '-S', GetoptLong::REQUIRED_ARGUMENT, _("Specify the list of images sizes to use instead of all specified in the theme (this is a comma-separated list)") ],
    [ '--thumbnails-per-row', '-T', GetoptLong::REQUIRED_ARGUMENT, _("Specify the amount of thumbnails per row in the thumbnails page (if applicable in theme)") ],
    [ '--optimize-for-32', '-o', GetoptLong::NO_ARGUMENT,       _("Resize images with optimized sizes for 3/2 aspect ratio rather than 4/3 (typical aspect ratio of pictures from non digital cameras are 3/2 when pictures from digital cameras are 4/3)") ],
    [ '--made-with',     '-n', GetoptLong::REQUIRED_ARGUMENT, _("Specify the HTML markup to use on the bottom of pages") ],
    [ '--empty-comments','-e', GetoptLong::NO_ARGUMENT,       _("Prefer empty comments over filename when creating new albums") ],

    [ '--mproc',         '-m', GetoptLong::REQUIRED_ARGUMENT, _("Specify the number of processors for multi-processors machines") ],

    [ '--for-gui',       '-g', GetoptLong::NO_ARGUMENT,       _("Do the minimum work to be able to see the album under the GUI (don't generate all thumbnails)") ],

    [ '--verbose-level', '-v', GetoptLong::REQUIRED_ARGUMENT, _("Set max verbosity level (0: errors, 1: warnings, 2: important messages, 3: other messages)") ],
    [ '--info-pipe',     '-i', GetoptLong::REQUIRED_ARGUMENT, _("Name a file where to write information about what's going on (used by the GUI)") ],
]

#- default values for some globals 
$switches = []
$stdout.sync = true
$no_identify = false
$ignore_videos = false
$forgui = false
$hardlinks_ok = true

def usage
    puts _("Usage: %s [OPTION]...") % File.basename($0)
    $options.each { |ary|
        printf " %3s, %-18s %s\n", ary[1], ary[0], ary[3]
    }
end

def handle_options
    parser = GetoptLong.new
    parser.set_options(*$options.collect { |ary| ary[0..2] })
    begin
        parser.each_option do |name, arg|
            case name
            when '--help'
                usage
                exit(0)

            when '--version'
                puts _("Booh version %s

Copyright (c) 2005 Guillaume Cottenceau.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.") % $VERSION

                exit(0)

            when '--source'
                $source = File.expand_path(arg.sub(%r|/$|, ''))
                if !File.directory?($source)
                    die _("Argument to --source must be a directory")
                end
            when '--destination'
                $dest = File.expand_path(arg.sub(%r|/$|, ''))
                if File.exists?($dest) && !File.directory?($dest)
                    die _("If --destination exists, it must be a directory")
                end
                if $dest != make_dest_filename($dest)
                    die _("Sorry, destination directory can't contain non simple alphanumeric characters.")
                end
#            when '--clean'
#                system("rm -rf #{$dest}")

            when '--theme'
                $theme = arg
            when '--config'
                arg = File.expand_path(arg)
                if File.readable?(arg)
                    $xmldoc = REXML::Document.new File.new(arg)
                    $mode = 'use_config'
                else
                    die _('Config file does not exist or is unreadable.')
                end
            when '--config-skel'
                arg = File.expand_path(arg)
                if File.exists?(arg)
                    if File.directory?(arg)
                        die _("Config skeleton file (%s) already exists and is a directory! Please change the filename.") % arg
                    else
                        msg 1, _("Config skeleton file already exists, backuping to %s.backup") % arg
                        File.rename(arg, "#{arg}.backup")
                    end
                end
                $config_writeto = arg
                $mode = 'gen_config'
            when '--merge-config'
                arg = File.expand_path(arg)
                if File.readable?(arg)
                    msg 2, _("Merge config notice: backuping current config file to %s.backup") % arg
                    $xmldoc = REXML::Document.new File.new(arg)
                    File.rename(arg, "#{arg}.backup")
                    $config_writeto = arg
                    $mode = 'merge_config'
                else
                    die _('Config file does not exist or is unreadable.')
                end
            when '--merge-config-onedir'
                arg = File.expand_path(arg)
                if File.readable?(arg)
                    msg 2, _("Merge config notice: backuping current config file to %s.backup") % arg
                    $xmldoc = REXML::Document.new File.new(arg)
                    File.rename(arg, "#{arg}.backup")
                    $config_writeto = arg
                    $mode = 'merge_config_onedir'
                else
                    die _('Config file does not exist or is unreadable.')
                end
            when '--merge-config-subdirs'
                arg = File.expand_path(arg)
                if File.readable?(arg)
                    msg 2, _("Merge config notice: backuping current config file to %s.backup") % arg
                    $xmldoc = REXML::Document.new File.new(arg)
                    File.rename(arg, "#{arg}.backup")
                    $config_writeto = arg
                    $mode = 'merge_config_subdirs'
                else
                    die _('Config file does not exist or is unreadable.')
                end
            when '--dir'
                arg = File.expand_path(arg)
                if !File.readable?(arg)
                    die _('Specified directory to merge with --dir is not readable')
                else
                    $onedir = arg
                end
            when '--use-config'
                arg = File.expand_path(arg)
                if File.readable?(arg)
                    msg 2, _("Use config notice: backuping current config file to %s.backup") % arg
                    $xmldoc = REXML::Document.new File.new(arg)
                    File.rename(arg, "#{arg}.backup")
                    $config_writeto = arg
                    $mode = 'use_config_changetheme'
                else
                    die _('Config file does not exist or is unreadable.')
                end

            when '--sizes'
                $limit_sizes = arg

            when '--thumbnails-per-row'
                $N_per_row = arg

            when '--optimize-for-32'
                $optimize_for_32 = true

            when '--made-with'
                $madewith = arg

            when '--empty-comments'
                $empty_comments = true

            when '--force'
                $force = true

            when '--mproc'
                $mproc = arg.to_i
                $pids = []

            when '--for-gui'
                $forgui = true

            when '--verbose-level'
                $verbose_level = arg.to_i

            when '--info-pipe'
                $info_pipe = File.open(arg, File::WRONLY)
                $info_pipe.sync = true
            end
        end
    rescue
        puts $!
        usage
        exit(1)
    end

    if !$source && $xmldoc
        $source = from_utf8($xmldoc.root.attributes['source'])
        $dest = from_utf8($xmldoc.root.attributes['destination'])
        $theme ||= $xmldoc.root.attributes['theme']
        $limit_sizes ||= $xmldoc.root.attributes['limit-sizes']
        if $mode == 'use_config' || $mode =~ /^merge_config/
            $optimize_for_32 = !$xmldoc.root.attributes['optimize-for-32'].nil?
            $N_per_row = $xmldoc.root.attributes['thumbnails-per-row']
            $madewith = $xmldoc.root.attributes['made-with']
        end
    end

    if $mode == 'merge_config_onedir' && !$onedir
        die _("Missing --dir for --merge-config-onedir")
    end
    if $mode == 'merge_config_subdirs' && !$onedir
        die _("Missing --dir for --merge-config-subdirs")
    end

    if !$source
        usage
        exit(0)
    end
    if !$dest
        die _("Missing --destination parameter.")
    end
    if !$theme
        $theme = 'simple'
    end

    select_theme($theme, $limit_sizes, $optimize_for_32, $N_per_row)

    if !$xmldoc
        additional_params = ''
        if $limit_sizes
            additional_params += "limit-sizes='#{$limit_sizes}'"
        end
        if $optimize_for_32
            additional_params += " optimize-for-32='true'"
        end
        if $N_per_row
            additional_params += " thumbnails-per-row='#{$N_per_row}'"
        end
        if $madewith
            additional_params += " made-with='#{$madewith}'"
        end
        $xmldoc = Document.new "<booh version='#{$VERSION}' source='#{utf8($source)}' destination='#{utf8($dest)}' theme='#{$theme}' #{additional_params}/>"
        $xmldoc << XMLDecl.new(XMLDecl::DEFAULT_VERSION, $CURRENT_CHARSET)
        $mode = 'gen_config'
    end

    if $mode == 'merge_config' || $mode == 'use_config_changetheme'
        $xmldoc.root.add_attribute('theme', $theme)
        $xmldoc.root.add_attribute('version', $VERSION)
        if $limit_sizes
            $xmldoc.root.add_attribute('limit-sizes', $limit_sizes)
        else
            $xmldoc.root.delete_attribute('limit-sizes')
        end
        if $optimize_for_32
            $xmldoc.root.add_attribute('optimize-for-32', 'true')
        else
            $xmldoc.root.delete_attribute('optimize-for-32')
        end
        if $N_per_row
            $xmldoc.root.add_attribute('thumbnails-per-row', $N_per_row)
        else
            $xmldoc.root.delete_attribute('thumbnails-per-row')
        end
        if $madewith
            $xmldoc.root.add_attribute('made-with', $madewith)
        else
            $xmldoc.root.delete_attribute('made-with')
        end
    end

    if $madewith
        $madewith = $madewith.gsub('%booh', '"http://zarb.org/~gc/html/booh.html"')
    end
end

def read_config
    $config = {}
end

def write_config
end

def info(value)
    if $info_pipe
        $info_pipe.puts(value)
    end
end

def die(value)
    if $info_pipe
        $info_pipe.puts("die: " + value)
    end
    die_ value
end

def check_installation
    if !system("which convert >/dev/null 2>/dev/null")
        die _("The program 'convert' is needed. Please install it. 
It is generally available with the 'ImageMagick' software package.")
    end
    if !system("which identify >/dev/null 2>/dev/null")
        msg 1, _("the program 'identify' is needed to get images sizes and EXIF data. Please install it.
It is generally available with the 'ImageMagick' software package.")
        $no_identify = true
    end
    missing = %w(transcode mencoder).delete_if { |prg| system("which #{prg} >/dev/null 2>/dev/null") }
    if missing != []
        msg 1, _("the following program(s) are needed to handle videos: '%s'. Videos will be ignored.") % missing.join(', ')
        $ignore_videos = true
    end
end

def replace_line(surround, keyword, line)
    begin
        contents = eval "$#{keyword}"
        line.sub!(/#{surround}#{keyword}#{surround}/, contents)
    rescue TypeError
        die _("No '%s' found for substitution") % keyword
    end
end

def build_html_skeletons
    $html_images     = File.open("#{$FPATH}/themes/#{$theme}/skeleton_image.html").readlines
    $html_thumbnails = File.open("#{$FPATH}/themes/#{$theme}/skeleton_thumbnails.html").readlines
    $html_index      = File.open("#{$FPATH}/themes/#{$theme}/skeleton_index.html").readlines
    for line in $html_images + $html_thumbnails + $html_index
        while line =~ /~~~(\w+)~~~/
            replace_line('~~~', $1, line)
        end
    end
end

def find_caption_value(xmldir, filename)
    if cap = xmldir.elements["*[@filename='#{utf8(filename)}']"].attributes['caption']
        return cap.gsub("\n", '<br/>')
    else
        return nil
    end
end

def find_captions(xmldir, images)
    return images.collect { |img| find_caption_value(xmldir, img) }
end

#- stolen from CVSspam
def urlencode(text)
  text.sub(/[^a-zA-Z0-9\-,.*_\/]/) do
    "%#{sprintf('%2X', $&[0])}"
  end
end

def all_images_sizes
    return $limit_sizes =~ /original/ ? $images_size + [ { 'name' => 'original' } ] : $images_size
end

def html_reload_to_thumbnails
    html_reload_to_thumbnails = $preferred_size_reloader.clone
    html_reload_to_thumbnails.gsub!(/~~theme~~/, $theme)
    html_reload_to_thumbnails.gsub!(/~~default_size~~/, $default_size['name'])
    html_reload_to_thumbnails.gsub!(/~~all_sizes~~/, all_images_sizes.collect { |s| "\"#{size2js(s['name'])}\"" }.join(', '))
    return html_reload_to_thumbnails
end

def discover_iterations(iterations, line)
    if line =~ /~~iterate(\d)_open(_max(\d+|N))?~~/
        for iter in iterations.values
            if iter['open']
                iter['open'] = false
                iter['close_wait'] = $1.to_i
            end
        end
        max = $3 == 'N' ? ($N_per_row || $default_N) : $3
        iterations[$1.to_i] = { 'open' => true, 'max' => max, 'opening' => '', 'closing' => '' }
        if $1.to_i == 1
            line.sub!(/.*/, '~~thumbnails~~')
        else
            line.sub!(/.*/, '')
        end
    elsif line =~ /~~iterate(\d)_close~~/
        iterations[$1.to_i]['open']  = false;
        iterations[$1.to_i]['close'] = true;
        line.sub!(/.*/, '')
    else
        for iter in iterations.values
            if iter['open']
                iter['opening'] += line
                line.sub!(/.*/, '')
            end
            if !iter['close'] && iter['close_wait'] && iterations[iter['close_wait']]['close']
                iter['closing'] += line
                line.sub!(/.*/, '')
            end
        end
    end
end

def reset_iterations(iterations)
    for iter in iterations.values
        iter['value'] = 0
    end
end

def run_iterations(iterations, amount)
    html = ''
    should_rerun = false
    for level in iterations.keys.sort
        if iterations[level]['value'] == 0
            html += iterations[level]['opening']
        elsif level == iterations.keys.max
            if !iterations[level]['max'] || iterations[level]['max'] && iterations[level]['value'] + amount <= iterations[level]['max'].to_i
                html += iterations[level]['opening']
            else
                should_rerun = true
            end
        end
        iterations[level]['value'] += amount
        if iterations[level]['max'] && iterations[level]['value'] > iterations[level]['max'].to_i
            iterations[level]['value'] = 0
            iterations[level-1]['value'] = 0
            html += iterations[level-1]['closing']
        end
    end
    if should_rerun
        return html + run_iterations(iterations, amount)
    else
        return html
    end
end

def close_iterations(iterations)
    html = ''
    for level in iterations.keys.sort.reverse
        html += iterations[level]['closing']
    end
    return html
end

def img_element(fullpath)
    if size = get_image_size(fullpath)
        sizespec = 'width="' + size[:x].to_s + '" height="' + size[:y].to_s + '"'
    else
        sizespec = ''
    end
    return '<img src="' + File.basename(fullpath) + '" ' + sizespec + ' class="image" alt="image"/>'
end

def size2js(name)
    return name.gsub(/-/, '')
end

def substitute_html_sizes(html, sizeobj, type)
    sizestrings = []
    if $images_size.length > 1 || (type == 'image' && $limit_sizes =~ /original/)
        for sizeobj2 in $images_size
            if sizeobj != sizeobj2
                if type == 'thumbnails'
                    sizestrings << '<a href="thumbnails-' + size2js(sizeobj2['name']) + '.html">' + sizename(sizeobj2['name']) + '</a>'
                else
                    sizestrings << '<a id="link' + size2js(sizeobj2['name']) + '">' + sizename(sizeobj2['name']) + '</a>'
                end
            else
                sizestrings << sizename(sizeobj2['name'])
            end
        end
        if type == 'image' && $limit_sizes =~ /original/
            sizestrings << '<a id="linkoriginal" target="newframe">' + sizename('original') + '</a>'
        end
    end
    html.sub!(/~~sizes~~(.+)~~/) { sizestrings.join($1) }
end

def xmldir2destdir(xmldir)
    return make_dest_filename(from_utf8(File.basename(xmldir.attributes['path'])))
end

def find_previous_album(xmldir)
    relative_pos = ''
    begin
        #- move to previous dir element if exists
        if prevelem = xmldir.previous_element_byname_notattr('dir', 'deleted')
            xmldir = prevelem
            relative_pos += '../' + xmldir2destdir(xmldir) + '/'
            child = nil
            #- after having moved to previous dir, we need to go down last subdir until the last one
            while child = xmldir.elements['dir']
                while nextchild = child.next_element_byname_notattr('dir', 'deleted')
                    child = nextchild
                end
                relative_pos += xmldir2destdir(child) + '/'
                xmldir = child
            end
        else
            #- previous dir doesn't exist, move to previous dir element if exists
            xmldir = xmldir.parent
            if xmldir.name == 'dir' && !xmldir.attributes['deleted']
                relative_pos += '../'
            else
                return nil
            end
        end
    end while !xmldir.child_byname_notattr('image', 'deleted') && !xmldir.child_byname_notattr('video', 'deleted')
    return File.reduce_path(relative_pos)
end

def find_next_album(xmldir)
    relative_pos = ''
    begin
        #- first child dir element (catches when initial xmldir has both thumbnails and subdirs)
        if firstchild = xmldir.child_byname_notattr('dir', 'deleted')
            xmldir = firstchild
            relative_pos += xmldir2destdir(xmldir) + '/'
        #- next brother
        elsif nextbro = xmldir.next_element_byname_notattr('dir', 'deleted')
            xmldir = nextbro
            relative_pos += '../' + xmldir2destdir(xmldir) + '/'
        else
            #- go up until we have a next brother or we are finished
            begin
                xmldir = xmldir.parent
                relative_pos += '../'
            end while xmldir && !xmldir.next_element_byname_notattr('dir', 'deleted')
            if xmldir
                xmldir = xmldir.next_element_byname('dir')
                relative_pos += '../' + xmldir2destdir(xmldir) + '/'
            else
                return nil
            end
        end
    end while !xmldir.child_byname_notattr('image', 'deleted') && !xmldir.child_byname_notattr('video', 'deleted')
    return File.reduce_path(relative_pos)
end

def sub_previous_next_album(previous_album, next_album, html)
    if previous_album
        html.gsub!(/~~previous_album~~/, '<a href="' + previous_album + 'thumbnails.html">' + utf8(_('previous album')) + '</a>')
        html.gsub!(/~~ifprevious_album\?~~(.+?)~~fi~~/) { $1 }
    else
        html.gsub!(/~~previous_album~~/, '')
        html.gsub!(/~~ifprevious_album\?~~(.+?)~~fi~~/, '')
    end
    if next_album
        html.gsub!(/~~next_album~~/, '<a href="' + next_album + 'thumbnails.html">' + utf8(_('next album')) + '</a>')
        html.gsub!(/~~ifnext_album\?~~(.+?)~~fi~~/) { $1 }
    else
        html.gsub!(/~~next_album~~/, '')
        html.gsub!(/~~ifnext_album\?~~(.+?)~~fi~~/, '')
    end
    return html
end

def walk_source_dir

    #- preprocess the path->dir, rexml is very slow with that; we seem to improve speed by 7%
    optxpath = {}
    $xmldoc.elements.each('//dir') { |elem|
        optxpath[elem.attributes['path']] = elem
    }

    examined_dirs = nil
    if $mode == 'merge_config_onedir'
        examined_dirs = [ $onedir ]
    elsif $mode == 'merge_config_subdirs'
        examined_dirs = `find '#{$onedir}' -type d -follow`.sort.collect { |v| v.chomp }.delete_if { |v| optxpath.has_key?(utf8(v)) }
    else
        examined_dirs = `find '#{$source}' -type d -follow`.sort.collect { |v| v.chomp }
        if $mode == 'merge_config'
            $xmldoc.elements.each('//dir') { |elem|
                if ! examined_dirs.include?(elem.attributes['path'])
                    msg 2, _("Merging config: removing directory %s from config, isn't on filesystem anymore") % elem.attributes['path']
                    elem.remove
                end
            }
        end
    end
    info("directories: #{examined_dirs.length}, sizes: #{$images_size.length}")

    examined_dirs.each { |dir|
        if File.basename(dir) =~ /^\./
            msg 1, _("Ignoring directory %s, begins with a dot (indicating a hidden directory)") % dir
            next
        end

        if dir =~ /'/
            die _("Source directory or sub-directories can't contain a single-quote character, sorry: %s") % dir
        end

        #- place xml document on proper node if exists, else create
        xmldir = optxpath[utf8(dir)]
        if $mode == 'use_config' || $mode == 'use_config_changetheme'
            if !xmldir || (xmldir.attributes['already-generated'] && !$force) || xmldir.attributes['deleted']
                info("walking: #{dir}|#{$source}, 0 elements")
                next
            end
        else
            if $mode == 'gen_config' || (($mode == 'merge_config' || $mode == 'merge_config_subdirs') && !xmldir)
                #- add the <dir..> element if necessary
                parent = File.dirname(dir)
                xmldir = $xmldoc.elements["//dir[@path='#{utf8(parent)}']"]
                if !xmldir
                    xmldir = $xmldoc.root
                end
                #- need to remove the already-generated mark of the parent because of the sub-albums page containing now one more element
                xmldir.delete_attribute('already-generated')
                xmldir = optxpath[utf8(dir)] = xmldir.add_element('dir', { 'path' => utf8(dir) })
            end
        end
        xmldir.delete_attribute('already-generated')

        #- read images/videos entries from config or from directories depending on mode
        entries = []
        if $mode == 'use_config' || $mode == 'use_config_changetheme'
            msg 2, _("Handling %s from config list...") % dir
            xmldir.elements.each { |element|
                if %w(image video).include?(element.name) && !element.attributes['deleted']
                    entries << from_utf8(element.attributes['filename'])
                end
            }
        else
            msg 2, _("Examining %s...") % dir
            entries = Dir.entries(dir).sort
            #- populate config in case of gen_config, add new files in case of merge_config
            for file in entries
                if file =~ /['"\[\]]/
                    msg 1, _("Ignoring %s, contains one of forbidden characters: '\"[]") % "#{dir}/#{file}"
                else
                    type = entry2type(file)
                    if type && !xmldir.elements["#{type}[@filename='#{utf8(file)}']"]
                        xmldir.add_element type, { "filename" => utf8(file), "caption" => $empty_comments ? '' : utf8cut(file.sub(/\.[^\.]+$/, ''), 18) }
                    end
                end
            end
            if $mode != 'gen_config'
                #- cleanup removed files from config and reread entries from config to get proper ordering
                entries = []
                xmldir.elements.each { |element|
                    fullpath = "#{dir}/#{from_utf8(element.attributes['filename'])}"
                    if %w(image video).include?(element.name)
                        if !File.readable?(fullpath)
                            msg 1, _("Config merge: removing %s from config; use the backup file to retrieve caption info if this was a mistake") % fullpath
                            xmldir.delete(element)
                        elsif !element.attributes['deleted']
                            entries << from_utf8(element.attributes['filename'])
                        end
                    end
                }
                #- if there is no more elements here, there is no album here anymore
                if !xmldir.child_byname_notattr('image', 'deleted') && !xmldir.child_byname_notattr('video', 'deleted')
                    xmldir.delete_attribute('thumbnails-caption')
                    xmldir.delete_attribute('thumbnails-captionfile')
                end
            end
        end
        images = entries.find_all { |e| entry2type(e) == 'image' }
        msg 3, _("\t%s images") % images.length
        videos = entries.find_all { |e| entry2type(e) == 'video' }
        msg 3, _("\t%s videos") % videos.length
        info("walking: #{dir}|#{$source}, #{images.length + videos.length} elements")

        dest_dir = make_dest_filename(dir.sub(/^#{Regexp.quote($source)}/, $dest))
        system("mkdir -p '#{dest_dir}'")

        #- pass through if there are no images and videos
        if images.size == 0 && videos.size == 0
            if !$forgui
                #- cleanup old images/videos, especially if this directory contained images/videos previously.
                themestuff = Dir.entries("#{$FPATH}/themes/#{$theme}").
                                find_all { |e| !%w(. .. skeleton_image.html skeleton_thumbnails.html skeleton_index.html metadata CVS).include?(e) }
                if $mode != 'gen_config'
                    rightful_images = []
                    if xmldir.attributes['thumbnails-caption']
                        rightful_images << 'thumbnails-thumbnail.jpg'
                    end
                    xmldir.elements.each('dir') { |child|
                        if child.attributes['deleted']
                            next
                        end
                        subdir = make_dest_filename(from_utf8(File.basename(child.attributes['path'])))
                        rightful_images << "thumbnails-#{subdir}.jpg"
                    }
                    to_del = Dir.entries(dest_dir).find_all { |e| !File.directory?("#{dest_dir}/#{e}") && !rightful_images.include?(e) } - themestuff
                    if to_del.size > 0
                        system("rm -f " + to_del.collect { |e| "#{dest_dir}/#{e}" }.join(' '))
                    end
                end
                
                #- copy any resource file that goes with the theme (css, images..)
                themestuff.each { |entry|
                    if !File.exists?("#{dest_dir}/#{entry}")
                        psys("cp '#{$FPATH}/themes/#{$theme}/#{entry}' '#{dest_dir}'")
                    end
                }
            end
            next
        end

        msg 2, _("Outputting in %s...") % dest_dir

        #- populate data structure with sizes from theme
        for sizeobj in $images_size
            fullscreen_images ||= {}
            fullscreen_images[sizeobj['name']] = []
            thumbnail_images ||= {}
            thumbnail_images[sizeobj['name']] = []
            thumbnail_videos ||= {}
            thumbnail_videos[sizeobj['name']] = []
        end
        #- a special dummy size to keep 'references' to thumbnails in case of panorama, because the GUI will use the regular thumbnails
        thumbnail_images['dont-delete-file-for-gui'] = []
        if $limit_sizes =~ /original/
            fullscreen_images['original'] = []
        end

        images.size >= 1 and msg 3, _("\tcreating images thumbnails...")

        #- create thumbnails for images
        images.each { |img|
            info("processing element")
            base_dest_img = dest_dir + '/' + make_dest_filename(img.sub(/\.[^\.]+$/, ''))
            if $forgui
                thumbnail_dest_img = base_dest_img + "-#{$default_size['thumbnails']}.jpg"
                gen_thumbnails_element("#{dir}/#{img}", xmldir, true, [ { 'filename' => thumbnail_dest_img, 'size' => $default_size['thumbnails'] } ])
            else
                todo = []
                elem = xmldir.elements["image[@filename='#{utf8(img)}']"]
                for sizeobj in $images_size
                    size_fullscreen = sizeobj['fullscreen']
                    size_thumbnails = sizeobj['thumbnails']
                    fullscreen_dest_img = base_dest_img + "-#{size_fullscreen}.jpg"
                    fullscreen_images[sizeobj['name']] << File.basename(fullscreen_dest_img)
                    todo << { 'filename' => fullscreen_dest_img, 'size' => size_fullscreen }
                    if pano = pano_amount(elem)
                        thumbnail_images['dont-delete-file-for-gui'] << File.basename(base_dest_img + "-#{size_thumbnails}.jpg")
                        size_thumbnails = size_thumbnails.sub(/(\d+)/) { ($1.to_i * pano).to_i }
                    end
                    thumbnail_dest_img = base_dest_img + "-#{size_thumbnails}.jpg"
                    thumbnail_images[sizeobj['name']] << File.basename(thumbnail_dest_img)
                    todo << { 'filename' => thumbnail_dest_img,  'size' => size_thumbnails }
                end
                gen_thumbnails_element("#{dir}/#{img}", xmldir, true, todo)
                if $limit_sizes =~ /original/
                    fullscreen_images['original'] << img
                end
                destimg = "#{dest_dir}/#{img}"
                if $limit_sizes =~ /original/ && !File.exists?(destimg)
                    if $hardlinks_ok
                        if ! sys("ln '#{dir}/#{img}' '#{destimg}'")
                            $hardlinks_ok = false
                        end
                    end
                    if ! $hardlinks_ok
                        psys("cp '#{dir}/#{img}' '#{destimg}'")
                    end
                end
            end
        }

        videos.size >= 1 and msg 3, _("\tcreating videos thumbnails...")

        #- create thumbnails for videos
        videos.each { |video|
            info("processing element")
            if $forgui
                thumbnail_dest_img = dest_dir + '/' + make_dest_filename(video.sub(/\.[^\.]+$/, '')) + "-#{$default_size['thumbnails']}.jpg"
                gen_thumbnails_element("#{dir}/#{video}", xmldir, true, [ { 'filename' => thumbnail_dest_img, 'size' => $default_size['thumbnails'] } ])
            else
                todo = []
                for sizeobj in $images_size
                    size_thumbnails = sizeobj['thumbnails']
                    thumbnail_dest_img = dest_dir + '/' + make_dest_filename(video.sub(/\.[^\.]+$/, '')) + "-#{size_thumbnails}.jpg"
                    thumbnail_videos[sizeobj['name']] << File.basename(thumbnail_dest_img)
                    todo << { 'filename' => thumbnail_dest_img, 'size' => size_thumbnails }
                end
                gen_thumbnails_element("#{dir}/#{video}", xmldir, true, todo)
            end
            destvideo = "#{dest_dir}/#{video}"
            if !File.exists?(destvideo)
                if $hardlinks_ok
                    if ! sys("ln '#{dir}/#{video}' '#{destvideo}'")
                        $hardlinks_ok = false
                    end
                end
                if ! $hardlinks_ok
                    psys("cp '#{dir}/#{video}' '#{destvideo}'")
                end
            end
        }

        if !$forgui
            themestuff = Dir.entries("#{$FPATH}/themes/#{$theme}").
                             find_all { |e| !%w(. .. skeleton_image.html skeleton_thumbnails.html skeleton_index.html metadata CVS).include?(e) }

            #- cleanup old images/videos (for when removing elements or sizes)
            all_elements = fullscreen_images.collect { |e| e[1] }.flatten.
                     concat(thumbnail_images.collect { |e| e[1] }.flatten).
                     concat(thumbnail_videos.collect { |e| e[1] }.flatten).
                     concat(videos)
            to_del = Dir.entries(dest_dir).find_all { |e| !File.directory?("#{dest_dir}/#{e}") && !all_elements.include?(e) && e !~ /^thumbnails-\w+\.jpg/ } - themestuff
            if to_del.size > 0
                msg 3, _("\tcleaning up: #{to_del.join(', ')}")
                system("rm -f " + to_del.collect { |e| "#{dest_dir}/#{e}" }.join(' '))
            end

            #- copy any resource file that goes with the theme (css, images..)
            themestuff.each { |entry|
                if !File.exists?("#{dest_dir}/#{entry}")
                    psys("cp '#{$FPATH}/themes/#{$theme}/#{entry}' '#{dest_dir}'")
                end
            }

            #- generate .htaccess file if needed
            if auth_user_file = xmldir.attributes['password-protect']
                msg 3, _("\tgenerating password protection file #{dest_dir}/.htaccess")
                ios = File.open("#{dest_dir}/.htaccess", "w")
                ios.write("AuthType Basic\nAuthName \"protected area\"\nAuthUserFile #{auth_user_file}\nrequire valid-user\n")
                ios.close
            end

            msg 3, _("\tgenerating HTML pages...")

            #- generate thumbnails.html (page with thumbnails)
            for sizeobj in $images_size
                info("processing size")
                html = $html_thumbnails.collect { |l| l.clone }
                iterations = {}
                for i in html
                    i.sub!(/~~run_slideshow~~/, images.size <= 1 ? '' : '<a href="image-' + size2js(sizeobj['name']) + '.html#run_slideshow=1">' + utf8(_('Run slideshow!'))+'</a>')
                    i.sub!(/~~title~~/, xmldir.attributes['thumbnails-caption'] || utf8(File.basename(dir)))
                    substitute_html_sizes(i, sizeobj, 'thumbnails')
                    discover_iterations(iterations, i)
                end
                html_thumbnails = ''
                reset_iterations(iterations)
                for file in entries
                    type = images.include?(file) ? 'image' : videos.include?(file) ? 'video' : nil
                    if type
                        if type == 'image' && elem = xmldir.elements["image[@filename='#{utf8(file)}']"]
                            if  pano = pano_amount(elem)
                                html_thumbnails += run_iterations(iterations, pano)
                                html_thumbnails.gsub!(/~~colspan~~/) { 'colspan="' + pano.ceil.to_s + '"' }
                            else
                                html_thumbnails += run_iterations(iterations, 1)
                                html_thumbnails.gsub!(/~~colspan~~/, '')
                            end
                        else 
                            html_thumbnails += run_iterations(iterations, 1)
                            html_thumbnails.gsub!(/~~colspan~~/, '')
                        end
                        if type == 'image'
                            index = images.index(file)
                            html_thumbnails.gsub!(/~~image_iteration~~/,
                                                  '<a href="image-' + size2js(sizeobj['name']) + '.html#current=' + fullscreen_images[sizeobj['name']][index] +
                                                      '" name="' + fullscreen_images[sizeobj['name']][index] + '">' +
                                                      img_element("#{dest_dir}/#{thumbnail_images[sizeobj['name']][index]}") + '</a>')
                            html_thumbnails.gsub!(/~~caption_iteration~~/,
                                                  find_caption_value(xmldir, images[index]) || utf8(images[index]))
                            html_thumbnails.gsub!(/~~ifimage\?~~(.+?)~~fi~~/) { $1 }
                            html_thumbnails.gsub!(/~~ifvideo\?~~(.+?)~~fi~~/, '')
                        elsif type == 'video'
                            index = videos.index(file)
                            if File.exists?("#{dest_dir}/#{thumbnail_videos[sizeobj['name']][index]}")
                                html_thumbnails.gsub!(/~~image_iteration~~/,
                                                      '<a href="' + videos[index] + '">' + img_element("#{dest_dir}/#{thumbnail_videos[sizeobj['name']][index]}") + '</a>')
                            else
                                html_thumbnails.gsub!(/~~image_iteration~~/,
                                                      '<a href="' + videos[index] + '">' + utf8(_("(no preview)")) + '</a>')
                            end
                            html_thumbnails.gsub!(/~~caption_iteration~~/,
                                                  find_caption_value(xmldir, videos[index]) || utf8(videos[index]))
                            html_thumbnails.gsub!(/~~ifimage\?~~(.+?)~~fi~~/, '')
                            html_thumbnails.gsub!(/~~ifvideo\?~~(.+?)~~fi~~/) { $1 }
                        end
                    end
                end
                html_thumbnails += close_iterations(iterations)
                for i in html
                    i.sub!(/~~thumbnails~~/, html_thumbnails)
                    i.gsub!(/~~theme~~/, $theme)
                    i.gsub!(/~~current_size~~/, sizeobj['name'])
                    i.gsub!(/~~current_size_js~~/, size2js(sizeobj['name']))
                    i.gsub!(/~~madewith~~/, $madewith || '')
                end
                ios = File.open("#{dest_dir}/thumbnails-#{size2js(sizeobj['name'])}.html", "w")
                ios.write(html)
                ios.close
            end

            info("finished processing sizes")

            #- generate "main" thumbnails.html page that will reload to correct size thanks to cookie
            ios = File.open("#{dest_dir}/thumbnails.html", "w")
            ios.write(html_reload_to_thumbnails)
            ios.close

            #- generate image.html (page with fullscreen images)
            if images.size > 0
                captions4js = find_captions(xmldir, images).collect { |e| e ? '"' + e.gsub('"', '\"') + '"' : '""' }.join(', ')

                for sizeobj in $images_size
                    html = $html_images.collect { |l| l.clone }
                    images4js = fullscreen_images[sizeobj['name']].collect { |e| "\"#{e}\"" }.join(', ')
                    otherimages4js = ''
                    othersizes = []
                    for sizeobj2 in all_images_sizes
                        if sizeobj != sizeobj2
                            otherimages4js += "var images_#{size2js(sizeobj2['name'])} = new Array(" + fullscreen_images[sizeobj2['name']].collect { |e| "\"#{e}\"" }.join(', ') + ")\n"
                            othersizes << "\"#{size2js(sizeobj2['name'])}\""
                        end
                    end
                    for i in html
                        i.gsub!(/~~images~~/, images4js)
                        i.gsub!(/~~other_images~~/, otherimages4js)
                        i.gsub!(/~~other_sizes~~/, othersizes.join(', '))
                        i.gsub!(/~~captions~~/, captions4js)
                        i.gsub!(/~~title~~/, xmldir.attributes['thumbnails-caption'] || utf8(File.basename(dir)))
                        i.gsub!(/~~thumbnails~~/, '<a href="thumbnails-' + size2js(sizeobj['name']) + '.html" id="thumbnails">' + utf8(_('return to thumbnails')) + '</a>')
                        i.gsub!(/~~theme~~/, $theme)
                        i.gsub!(/~~current_size~~/, size2js(sizeobj['name']))
                        i.gsub!(/~~madewith~~/, $madewith || '')
                        substitute_html_sizes(i, sizeobj, 'image')
                    end
                    ios = File.open("#{dest_dir}/image-#{size2js(sizeobj['name'])}.html", "w")
                    ios.write(html)
                    ios.close
                end
            end
        end
    }

    msg 3, ''

    #- add attributes to <dir..> elements needing so
    if $mode != 'use_config'
        msg 3, _("\tfixating configuration file...")
        $xmldoc.elements.each('//dir') { |element|
            path = captionpath = element.attributes['path']
            descendant_element = element.elements['descendant::image'] || element.elements['descendant::video']
            if !descendant_element
                msg 3, _("\t\tremoving %s, no element in it") % path
                element.remove  #- means we have a directory with nothing interesting in it
            else
                captionfile = "#{descendant_element.parent.attributes['path']}/#{descendant_element.attributes['filename']}"
                basename = File.basename(path)
                if element.elements['dir']
                    if !element.attributes['subdirs-caption']
                        element.add_attribute('subdirs-caption', basename)
                    end
                    if !element.attributes['subdirs-captionfile']
                        element.add_attribute('subdirs-captionfile', captionfile)
                    end
                end
                if element.child_byname_notattr('image', 'deleted') || element.child_byname_notattr('video', 'deleted')
                    if !element.attributes['thumbnails-caption']
                        element.add_attribute('thumbnails-caption', basename)
                    end
                    if !element.attributes['thumbnails-captionfile']
                        element.add_attribute('thumbnails-captionfile', captionfile)
                    end
                end
            end
        }
    end

    #- write down to disk config if necessary
    if $config_writeto
        ios = File.open($config_writeto, "w")
        $xmldoc.write(ios, 0)
        ios.close
    end

    if $forgui
        msg 3, _(" completed necessary stuff for GUI, exiting.")
        return
    end

    #- second pass to create index.html files and previous/next links
    info("creating index.html")
    msg 3, _("\trescanning directories to generate all 'index.html' files...")

    #- recompute the memoization because elements mights have been removed (the ones with no element in them)
    optxpath = {}
    $xmldoc.elements.each('//dir') { |elem|
        optxpath[elem.attributes['path']] = elem
    }

    examined_dirs.each { |dir|
        info("index.html: #{dir}|#{$source}")

        xmldir = optxpath[utf8(dir)]
        if !xmldir || (xmldir.attributes['already-generated'] && !$force) || xmldir.attributes['deleted']
            next
        end
        dest_dir = make_dest_filename(dir.sub(/^#{Regexp.quote($source)}/, $dest))

        previous_album = find_previous_album(xmldir)
        next_album = find_next_album(xmldir)

        if xmldir.elements['dir']
            html = $html_index.collect { |l| l.clone }
            iterations = {}
            for i in html
                caption = xmldir.attributes['subdirs-caption']
                i.gsub!(/~~title~~/, caption)
                if xmldir.parent.name == 'dir'
                    nav = ''
                    path = '..'
                    parent = xmldir.parent
                    while parent.name == 'dir'
                        parentcaption = parent.attributes['subdirs-caption']
                        nav = "<a href='#{path}/index.html'>#{parentcaption}</a> #{utf8(_(" > "))} #{nav}"
                        path += '/..'
                        parent = parent.parent
                    end
                    i.gsub!(/~~ifnavigation\?~~(.+?)~~fi~~/) { $1 }
                    i.gsub!(/~~navigation~~/, nav + caption)
                else
                    i.gsub!(/~~ifnavigation\?~~(.+?)~~fi~~/, '')
                end
                discover_iterations(iterations, i)
            end
            
            html_index = ''
            reset_iterations(iterations)
            
            #- deal with "current" album (directs to "thumbnails" page)
            if xmldir.attributes['thumbnails-caption']
                thumbnail = "#{dest_dir}/thumbnails-thumbnail.jpg"
                gen_thumbnails_subdir(from_utf8(xmldir.attributes['thumbnails-captionfile']), xmldir, false,
                                      [ { 'filename' => thumbnail, 'size' => $albums_thumbnail_size } ], 'thumbnails')
                html_index += run_iterations(iterations, 1)
                html_index.gsub!(/~~image_iteration~~/, "<a href='thumbnails.html'>" + img_element(thumbnail) + '</a>')
                html_index.gsub!(/~~caption_iteration~~/, xmldir.attributes['thumbnails-caption'])
            end

            #- deal with sub-albums (direct to subdirs/index.html pages)
            xmldir.elements.each('dir') { |child|
                if child.attributes['deleted']
                    next
                end
                subdir = make_dest_filename(from_utf8(File.basename(child.attributes['path'])))
                thumbnail = "#{dest_dir}/thumbnails-#{subdir}.jpg"
                html_index += run_iterations(iterations, 1)
                captionfile, caption = find_subalbum_caption_info(child)
                gen_thumbnails_subdir(captionfile, child, false,
                                      [ { 'filename' => thumbnail, 'size' => $albums_thumbnail_size } ], find_subalbum_info_type(child))
                html_index.gsub!(/~~caption_iteration~~/, caption)
                html_index.gsub!(/~~image_iteration~~/, "<a href='#{make_dest_filename(subdir)}/index.html'>" + img_element(thumbnail) + '</a>')
            }

            html_index += close_iterations(iterations)

            for i in html
                i.gsub!(/~~thumbnails~~/, html_index)
                i.gsub!(/~~madewith~~/, $madewith || '')
            end
            
        else
            html = html_reload_to_thumbnails
        end

        ios = File.open("#{dest_dir}/index.html", "w")
        ios.write(html)
        ios.close

        #- substitute "return to albums" and previous/next correctly
        if xmldir.child_byname_notattr('image', 'deleted') || xmldir.child_byname_notattr('video', 'deleted')
            for sizeobj in $images_size
                substInFile("#{dest_dir}/thumbnails-#{size2js(sizeobj['name'])}.html") { |line|
                    sub_previous_next_album(previous_album, next_album, line)
                    if xmldir.elements['dir']
                        line.sub!(/~~return_to_albums~~/, '<a href="index.html">' + utf8(_('return to albums')) + '</a>')
                    else
                        if xmldir.parent.name == 'dir'
                            line.sub!(/~~return_to_albums~~/, '<a href="../index.html">' + utf8(_('return to albums')) + '</a>')
                        else
                            line.sub!(/~~return_to_albums~~/, '')
                        end
                    end
                    line
                }
                if xmldir.child_byname_notattr('image', 'deleted')
                    substInFile("#{dest_dir}/image-#{size2js(sizeobj['name'])}.html") { |line|
                        sub_previous_next_album(previous_album, next_album, line)
                    }
                end
            end
        end
    }

    msg 3, _(" all done.")
end

handle_options
read_config
check_installation

build_html_skeletons

walk_source_dir