enable long (>260) and reserved name paths on Windows

Allows to write files and directories with paths longer
than 260 characters or with Windows reserved name components
(e.g. aux, nul, prn) in the store and in the working copy.

Works on NTFS and FAT32 file systems.

Note that explorer and most other tools can't copy nor
delete directories containing paths >260 or paths with
reserved name components. If you want to use explorer for
deleting and copying of such repos, then encode the paths
in the store to make them shorter and not contain reserved
names.

diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py
--- a/mercurial/cmdutil.py
+++ b/mercurial/cmdutil.py
@@ -220,9 +220,8 @@
         return pat
     if hasattr(pat, 'read') and 'r' in mode:
         return pat
-    return open(make_filename(repo, pat, node, total, seqno, revwidth,
-                              pathname),
-                mode)
+    n = make_filename(repo, pat, node, total, seqno, revwidth, pathname)
+    return open(util.longpath(n), mode)
 
 def match(repo, pats=[], opts={}, globbed=False, default='relpath'):
     if not globbed and default == 'relpath':
@@ -359,8 +358,10 @@
                      repo.pathto(prevsrc, cwd)))
             return
 
+        lp = util.longpath
+        lptarget = lp(target)
         # check for overwrites
-        exists = os.path.exists(target)
+        exists = os.path.exists(lptarget)
         if (not after and exists or after and state in 'mn'):
             if not opts['force']:
                 ui.warn(_('%s: not overwriting - file exists\n') %
@@ -373,10 +374,10 @@
         elif not dryrun:
             try:
                 if exists:
-                    os.unlink(target)
-                targetdir = os.path.dirname(target) or '.'
-                if not os.path.isdir(targetdir):
-                    os.makedirs(targetdir)
+                    os.unlink(lptarget)
+                lptargetdir = lp(os.path.dirname(lptarget) or '.')
+                if not os.path.isdir(lptargetdir):
+                    os.makedirs(lptargetdir)
                 util.copyfile(src, target)
             except IOError, inst:
                 if inst.errno == errno.ENOENT:
diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py
--- a/mercurial/dirstate.py
+++ b/mercurial/dirstate.py
@@ -284,7 +284,7 @@
         'mark a file normal and clean'
         self._dirty = True
         self._changepath(f, 'n', True)
-        s = os.lstat(self._join(f))
+        s = os.lstat(util.longpath(self._join(f)))
         self._map[f] = ('n', s.st_mode, s.st_size, s.st_mtime, 0)
         if f in self._copymap:
             del self._copymap[f]
@@ -530,6 +530,7 @@
         supported = self._supported
         _join = self._join
         known = {'.hg': 1}
+        lp = util.longpath
 
         # recursion free walker, faster than os.walk.
         def findfiles(s):
@@ -583,7 +584,7 @@
             nn = self.normalize(nf)
             f = _join(ff)
             try:
-                st = lstat(f)
+                st = lstat(lp(f))
             except OSError, inst:
                 found = False
                 for fn in dc:
@@ -636,6 +637,7 @@
         radd = removed.append
         dadd = deleted.append
         cadd = clean.append
+        lp = util.longpath
 
         for src, fn, st in self.walk(match, listunknown, listignored):
             if fn not in dmap:
@@ -652,7 +654,7 @@
                 nonexistent = True
                 if not st:
                     try:
-                        st = lstat(_join(fn))
+                        st = lstat(lp(_join(fn)))
                     except OSError, inst:
                         if inst.errno not in (errno.ENOENT, errno.ENOTDIR):
                             raise
@@ -666,7 +668,7 @@
             # check the common case first
             if state == 'n':
                 if not st:
-                    st = lstat(_join(fn))
+                    st = lstat(lp(_join(fn)))
                 if (size >= 0 and
                     (size != st.st_size
                      or ((mode ^ st.st_mode) & 0100 and self._checkexec))
diff --git a/mercurial/hg.py b/mercurial/hg.py
--- a/mercurial/hg.py
+++ b/mercurial/hg.py
@@ -129,7 +129,7 @@
     dest = localpath(dest)
     source = localpath(source)
 
-    if os.path.exists(dest):
+    if os.path.exists(util.longpath(dest)):
         raise util.Abort(_("destination '%s' already exists") % dest)
 
     class DirCleanup(object):
@@ -165,17 +165,19 @@
 
         if copy:
             def force_copy(src, dst):
-                if not os.path.exists(src):
+                if not os.path.exists(util.longpath(src)):
                     # Tolerate empty source repository and optional files
                     return
                 util.copyfiles(src, dst)
 
             src_store = os.path.realpath(src_repo.spath)
-            if not os.path.exists(dest):
-                os.mkdir(dest)
+            lp = util.longpath
+            lpdest = lp(dest)
+            if not os.path.exists(lpdest):
+                os.mkdir(lpdest)
             try:
                 dest_path = os.path.realpath(os.path.join(dest, ".hg"))
-                os.mkdir(dest_path)
+                os.mkdir(lp(dest_path))
             except OSError, inst:
                 if inst.errno == errno.EEXIST:
                     dir_cleanup.close()
@@ -188,7 +190,7 @@
                 # copy the dummy changelog
                 force_copy(src_repo.join("00changelog.i"), dummy_changelog)
                 dest_store = os.path.join(dest_path, "store")
-                os.mkdir(dest_store)
+                os.mkdir(lp(dest_store))
             else:
                 dest_store = dest_path
             # copy the requires file
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -25,14 +25,16 @@
         self.opener = util.opener(self.path)
         self.wopener = util.opener(self.root)
 
-        if not os.path.isdir(self.path):
+        lp = util.longpath
+        if not os.path.isdir(lp(self.path)):
             if create:
-                if not os.path.exists(path):
-                    os.mkdir(path)
-                os.mkdir(self.path)
+                lppath = lp(path)
+                if not os.path.exists(lppath):
+                    os.mkdir(lppath)
+                os.mkdir(lp(self.path))
                 requirements = ["revlogv1"]
                 if parentui.configbool('format', 'usestore', True):
-                    os.mkdir(os.path.join(self.path, "store"))
+                    os.mkdir(lp(os.path.join(self.path, "store")))
                     requirements.append("store")
                     # create an invalid changelog
                     self.opener("00changelog.i", "a").write(
@@ -1084,7 +1086,7 @@
             for f in list:
                 p = self.wjoin(f)
                 try:
-                    st = os.lstat(p)
+                    st = os.lstat(util.longpath(p))
                 except:
                     self.ui.warn(_("%s does not exist!\n") % f)
                     rejected.append(f)
@@ -1162,7 +1164,7 @@
     def copy(self, source, dest):
         wlock = None
         try:
-            p = self.wjoin(dest)
+            p = util.longpath(self.wjoin(dest))
             if not (os.path.exists(p) or os.path.islink(p)):
                 self.ui.warn(_("%s does not exist!\n") % dest)
             elif not (os.path.isfile(p) or os.path.islink(p)):
diff --git a/mercurial/lock.py b/mercurial/lock.py
--- a/mercurial/lock.py
+++ b/mercurial/lock.py
@@ -104,7 +104,7 @@
         try:
             l = lock(self.f + '.break')
             l.trylock()
-            os.unlink(self.f)
+            os.unlink(util.longpath(self.f))
             l.release()
         except (LockHeld, LockUnavailable):
             return locker
diff --git a/mercurial/osutil.py b/mercurial/osutil.py
--- a/mercurial/osutil.py
+++ b/mercurial/osutil.py
@@ -1,4 +1,4 @@
-import os, stat
+import os, stat, util
 
 def _mode_to_kind(mode):
     if stat.S_ISREG(mode): return stat.S_IFREG
@@ -26,10 +26,13 @@
     '''
     result = []
     prefix = path + os.sep
-    names = os.listdir(path)
+    names = os.listdir(util.longpath(path)) # returns unicode strings on Windows
     names.sort()
     for fn in names:
-        st = os.lstat(prefix + fn)
+        def shrink(unicodestring):
+            return ''.join([chr(ord(c)) for c in unicodestring])
+        fn = shrink(fn)
+        st = os.lstat(util.longpath(prefix + fn))
         if stat:
             result.append((fn, _mode_to_kind(st.st_mode), st))
         else:
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -359,7 +359,11 @@
 
     def parseindex(self, fp, inline):
         try:
-            size = util.fstat(fp).st_size
+            if callable(getattr(fp, "size", None)):
+               # assuming util_win32.posixfile_nt
+               size = fp.size()
+            else:
+               size = util.fstat(fp).st_size
         except AttributeError:
             size = 0
 
diff --git a/mercurial/transaction.py b/mercurial/transaction.py
--- a/mercurial/transaction.py
+++ b/mercurial/transaction.py
@@ -12,7 +12,7 @@
 # of the GNU General Public License, incorporated herein by reference.
 
 from i18n import _
-import os
+import os, util
 
 class transaction(object):
     def __init__(self, report, opener, journal, after=None, createmode=None):
@@ -26,15 +26,16 @@
         self.map = {}
         self.journal = journal
 
-        self.file = open(self.journal, "w")
+        lp = util.longpath
+        self.file = open(lp(self.journal), "w")
         if createmode is not None:
-            os.chmod(self.journal, createmode & 0666)
+            os.chmod(lp(self.journal), createmode & 0666)
 
     def __del__(self):
         if self.journal:
             if self.entries: self.abort()
             self.file.close()
-            try: os.unlink(self.journal)
+            try: os.unlink(util.longpath(self.journal))
             except: pass
 
     def add(self, file, offset, data=None):
@@ -74,7 +75,7 @@
         if self.after:
             self.after()
         else:
-            os.unlink(self.journal)
+            os.unlink(util.longpath(self.journal))
         self.journal = None
 
     def abort(self):
@@ -93,6 +94,7 @@
         self.report(_("rollback completed\n"))
 
 def rollback(opener, file):
+    lp = util.longpath
     files = {}
     for l in open(file).readlines():
         f, o = l.split('\0')
@@ -103,6 +105,6 @@
             opener(f, "a").truncate(int(o))
         else:
             fn = opener(f).name
-            os.unlink(fn)
-    os.unlink(file)
+            os.unlink(lp(fn))
+    os.unlink(lp(file))
 
diff --git a/mercurial/util.py b/mercurial/util.py
--- a/mercurial/util.py
+++ b/mercurial/util.py
@@ -650,8 +650,10 @@
 
 def rename(src, dst):
     """forcibly rename a file"""
+    lpsrc = longpath(src)
+    lpdst = longpath(dst)
     try:
-        os.rename(src, dst)
+        os.rename(lpsrc, lpdst)
     except OSError, err: # FIXME: check err (EEXIST ?)
         # on windows, rename to existing file is not allowed, so we
         # must delete destination first. but if file is open, unlink
@@ -659,46 +661,52 @@
         # happens immediately even for open files, so we create
         # temporary file, delete it, rename destination to that name,
         # then delete that. then rename is safe to do.
-        fd, temp = tempfile.mkstemp(dir=os.path.dirname(dst) or '.')
+        fd, temp = mkstemp(dir=os.path.dirname(dst) or '.')
         os.close(fd)
-        os.unlink(temp)
-        os.rename(dst, temp)
-        os.unlink(temp)
-        os.rename(src, dst)
+        lptemp = longpath(temp)
+        os.unlink(lptemp)
+        os.rename(lpdst, lptemp)
+        os.unlink(lptemp)
+        os.rename(lpsrc, lpdst)
 
 def unlink(f):
     """unlink and remove the directory if it is empty"""
-    os.unlink(f)
+    os.unlink(longpath(f))
     # try removing directories that might now be empty
     try:
-        os.removedirs(os.path.dirname(f))
+        os.removedirs(longpath(os.path.dirname(f)))
     except OSError:
         pass
 
 def copyfile(src, dest):
     "copy a file, preserving mode"
-    if os.path.islink(src):
+    lpsrc = longpath(src)
+    lpdest = longpath(dest)
+    if os.path.islink(lpsrc):
         try:
-            os.unlink(dest)
+            os.unlink(lpdest)
         except:
             pass
-        os.symlink(os.readlink(src), dest)
+        os.symlink(os.readlink(lpsrc), lpdest)
     else:
         try:
-            shutil.copyfile(src, dest)
-            shutil.copymode(src, dest)
+            shutil.copyfile(lpsrc, lpdest)
+            shutil.copymode(lpsrc, lpdest)
         except shutil.Error, inst:
             raise Abort(str(inst))
 
 def copyfiles(src, dst, hardlink=None):
     """Copy a directory tree using hardlinks if possible"""
 
+    lpsrc = longpath(src)
+    lpdst = longpath(dst)
+
     if hardlink is None:
-        hardlink = (os.stat(src).st_dev ==
-                    os.stat(os.path.dirname(dst)).st_dev)
+        hardlink = (os.stat(lpsrc).st_dev ==
+                    os.stat(longpath(os.path.dirname(dst))).st_dev)
 
-    if os.path.isdir(src):
-        os.mkdir(dst)
+    if os.path.isdir(lpsrc):
+        os.mkdir(lpdst)
         for name, kind in osutil.listdir(src):
             srcname = os.path.join(src, name)
             dstname = os.path.join(dst, name)
@@ -706,12 +714,12 @@
     else:
         if hardlink:
             try:
-                os_link(src, dst)
+                os_link(lpsrc, lpdst)
             except (IOError, OSError):
                 hardlink = False
-                shutil.copy(src, dst)
+                shutil.copy(lpsrc, lpdst)
         else:
-            shutil.copy(src, dst)
+            shutil.copy(lpsrc, lpdst)
 
 class path_auditor(object):
     '''ensure that a filesystem path contains no banned components.
@@ -769,7 +777,7 @@
         self.auditeddir.update(prefixes)
 
 def _makelock_file(info, pathname):
-    ld = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL)
+    ld = os.open(longpath(pathname), os.O_CREAT | os.O_WRONLY | os.O_EXCL)
     os.write(ld, info)
     os.close(ld)
 
@@ -977,10 +985,36 @@
 def lookup_reg(key, name=None, scope=None):
     return None
 
+def makedirs(name, mode=None):
+    """recursive directory creation with parent mode inheritance"""
+    try:
+        os.mkdir(name)
+        if mode is not None:
+            os.chmod(name, mode)
+        return
+    except OSError, err:
+        if err.errno == errno.EEXIST:
+            return
+        if err.errno != errno.ENOENT:
+            raise
+    parent = os.path.abspath(os.path.dirname(name))
+    makedirs(parent, mode)
+    makedirs(name, mode)
+
 # Platform specific variants
 if os.name == 'nt':
     import msvcrt
     nulldev = 'NUL:'
+
+    def mkstemp(prefix=None, dir=None):
+        # tempfile.mkstemp fails for long path dirs on Windows
+        # just always create the tempfile in the system default dir
+        # (ignore parameter dir, which might be a long path)
+        if prefix == None:
+           res = tempfile.mkstemp()
+        else:
+           res = tempfile.mkstemp(prefix=prefix)
+        return res
 
     class winstdout:
         '''stdout on windows misbehaves if sent through a pipe'''
@@ -1084,6 +1118,23 @@
     def localpath(path):
         return path.replace('/', '\\')
 
+    _longpathprefix = "\\\\?\\"
+    def longpath(path):
+        '''convert path to a Windows long path
+        needed to call Windows api with paths longer than 260'''
+        if path.startswith(_longpathprefix):
+            res = path
+        else:
+            path = path.replace('/', '\\').replace('\\.\\', '\\')
+            if path[-1] == '.':
+                path = path[:-1]
+            if not os.path.isabs(path):
+                path = os.path.abspath(path)
+            def expand(s):
+               return u''.join([unichr(ord(c)) for c in s])
+            res = expand(_longpathprefix + path)
+        return res
+
     def normpath(path):
         return pconvert(os.path.normpath(path))
 
@@ -1173,6 +1224,8 @@
 else:
     nulldev = '/dev/null'
 
+    mkstemp = tempfile.mkstemp
+
     def rcfiles(path):
         rcs = [os.path.join(path, 'hgrc')]
         rcdir = os.path.join(path, 'hgrc.d')
@@ -1251,6 +1304,9 @@
         return path
 
     def localpath(path):
+        return path
+
+    def longpath(path):
         return path
 
     normpath = os.path.normpath
@@ -1389,13 +1445,13 @@
     Returns the name of the temporary file.
     """
     d, fn = os.path.split(name)
-    fd, temp = tempfile.mkstemp(prefix='.%s-' % fn, dir=d)
+    fd, temp = mkstemp(prefix='.%s-' % fn, dir=d)    
     os.close(fd)
     # Temporary files are created with mode 0600, which is usually not
     # what we want.  If the original file already exists, just copy
     # its mode.  Otherwise, manually obey umask.
     try:
-        st_mode = os.lstat(name).st_mode & 0777
+        st_mode = os.lstat(longpath(name)).st_mode & 0777
     except OSError, inst:
         if inst.errno != errno.ENOENT:
             raise
@@ -1403,7 +1459,7 @@
         if st_mode is None:
             st_mode = ~_umask
         st_mode &= 0666
-    os.chmod(temp, st_mode)
+    os.chmod(longpath(temp), st_mode)
     if emptyok:
         return temp
     try:
@@ -1421,7 +1477,7 @@
         ifp.close()
         ofp.close()
     except:
-        try: os.unlink(temp)
+        try: os.unlink(longpath(temp))
         except: pass
         raise
     return temp
@@ -1447,25 +1503,9 @@
     def __del__(self):
         if not self.closed:
             try:
-                os.unlink(self.temp)
+                os.unlink(longpath(self.temp))
             except: pass
             posixfile.close(self)
-
-def makedirs(name, mode=None):
-    """recursive directory creation with parent mode inheritance"""
-    try:
-        os.mkdir(name)
-        if mode is not None:
-            os.chmod(name, mode)
-        return
-    except OSError, err:
-        if err.errno == errno.EEXIST:
-            return
-        if err.errno != errno.ENOENT:
-            raise
-    parent = os.path.abspath(os.path.dirname(name))
-    makedirs(parent, mode)
-    makedirs(name, mode)
 
 class opener(object):
     """Open files relative to a base directory
diff --git a/mercurial/util_win32.py b/mercurial/util_win32.py
--- a/mercurial/util_win32.py
+++ b/mercurial/util_win32.py
@@ -15,7 +15,7 @@
 
 import errno, os, sys, pywintypes, win32con, win32file, win32process
 import cStringIO, winerror
-import osutil
+import osutil, util
 from win32com.shell import shell,shellcon
 
 class WinError:
@@ -146,13 +146,15 @@
                          self.win_strerror)
 
 def os_link(src, dst):
+    lpsrc = util.longpath(src)
+    lpdst = util.longpath(dst)
     try:
-        win32file.CreateHardLink(dst, src)
+        win32file.CreateHardLink(lpdst, lpsrc)
         # CreateHardLink sometimes succeeds on mapped drives but
         # following nlinks() returns 1. Check it now and bail out.
         if nlinks(src) < 2:
             try:
-                win32file.DeleteFile(dst)
+                win32file.DeleteFileW(lpdst)
             except:
                 pass
             # Fake hardlinking error
@@ -164,7 +166,7 @@
 def nlinks(pathname):
     """Return number of hardlinks for the given file."""
     try:
-        fh = win32file.CreateFile(pathname,
+        fh = win32file.CreateFileW(util.longpath(pathname),
             win32file.GENERIC_READ, win32file.FILE_SHARE_READ,
             None, win32file.OPEN_EXISTING, 0, None)
         res = win32file.GetFileInformationByHandle(fh)
@@ -268,7 +270,8 @@
 
     def __init__(self, name, mode='rb'):
         self.closed = False
-        self.name = name
+        # use long file names. Must be absolute and use backslash path sep only
+        self.name = '\\\\?\\' + name.replace('/','\\')
         self.mode = mode
         access = 0
         if 'r' in mode or '+' in mode:
@@ -282,7 +285,8 @@
         else:
             creation = win32file.CREATE_ALWAYS
         try:
-            self.handle = win32file.CreateFile(name,
+            # use ..W function in order to get \\?\.. long paths through
+            self.handle = win32file.CreateFileW(self.name,
                                                access,
                                                win32file.FILE_SHARE_READ |
                                                win32file.FILE_SHARE_WRITE |
@@ -292,7 +296,7 @@
                                                win32file.FILE_ATTRIBUTE_NORMAL,
                                                0)
         except pywintypes.error, err:
-            raise WinIOError(err, name)
+            raise WinIOError(err, self.name)
 
     def __iter__(self):
         for line in self.read().splitlines(True):
@@ -359,6 +363,9 @@
         except pywintypes.error, err:
             raise WinIOError(err)
 
+    def size(self):
+        return win32file.GetFileSize(self.handle)
+
 getuser_fallback = win32api.GetUserName
 
 def set_signal_handler_win32():
@@ -369,3 +376,20 @@
     def handler(event):
         win32process.ExitProcess(1)
     win32api.SetConsoleCtrlHandler(handler)
+
+def makedirs(name, mode=None):
+    """recursive directory creation for Windows long paths. Parameter mode is unused"""
+    if not name.startswith('\\\\?\\'):
+        name = '\\\\?\\' + name.replace('/', '\\')
+    try:
+        win32file.CreateDirectoryW(name, None)
+        return
+    except pywintypes.error, details:
+        err = WinOSError(details)
+        if err.errno == errno.EEXIST:
+            return
+        if err.errno != errno.ENOENT:
+            raise
+        parent = os.path.dirname(name)
+        makedirs(parent, mode)
+        makedirs(name, mode)