index: Entries are now using flags internally, instead of reducing the flag information to just the stage ( just to be closer to the git-original )

Author: Byron

lib/git/index/base.py

@@ -9,15 +9,29 @@
 import tempfile
 import os
 import sys
-import stat
 import subprocess
 import glob
 from cStringIO import StringIO
 
-from typ import *
+from stat import (
+					S_ISLNK,
+					S_ISDIR,
+					S_IFMT,
+					S_IFDIR,
+					S_IFLNK,
+					S_IFREG
+				)
+
+from typ import (
+					BaseIndexEntry, 
+					IndexEntry, 
+					CE_NAMEMASK,
+					CE_STAGESHIFT
+				)
+
 from util import (
 					TemporaryFileSwap,
-					clear_cache, 
+					post_clear_cache, 
 					default_index,
 					pack, 
 					unpack
@@ -75,7 +89,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
 	"""
 	__slots__ = ( "repo", "version", "entries", "_extension_data", "_file_path" )
 	_VERSION = 2			# latest version we support
-	S_IFGITLINK = 0160000
+	S_IFGITLINK = 0160000	# a submodule
 
 	def __init__(self, repo, file_path=None):
 		"""
@@ -141,12 +155,12 @@ def _read_entry(cls, stream):
 		mtime = unpack(">8s", stream.read(8))[0]
 		(dev, ino, mode, uid, gid, size, sha, flags) = \
 			unpack(">LLLLLL20sH", stream.read(20 + 4 * 6 + 2))
-		path_size = flags & 0x0fff
+		path_size = flags & CE_NAMEMASK
 		path = stream.read(path_size)
 
 		real_size = ((stream.tell() - beginoffset + 8) & ~7)
 		data = stream.read((beginoffset + real_size) - stream.tell())
-		return IndexEntry((mode, binascii.hexlify(sha), flags >> 12, path, ctime, mtime, dev, ino, uid, gid, size))
+		return IndexEntry((mode, binascii.hexlify(sha), flags, path, ctime, mtime, dev, ino, uid, gid, size))
 
 	@classmethod
 	def _read_header(cls, stream):
@@ -198,7 +212,7 @@ def _serialize(self, stream, ignore_tree_extension_data=False):
 
 		# body
 		entries_sorted = self.entries.values()
-		entries_sorted.sort(key=lambda e: (e[3], e[2]))		# use path/stage as sort key
+		entries_sorted.sort(key=lambda e: (e[3], e.stage))		# use path/stage as sort key
 		for entry in entries_sorted:
 			self._write_cache_entry(stream, entry)
 		# END for each entry
@@ -226,17 +240,18 @@ def _serialize(self, stream, ignore_tree_extension_data=False):
 	def _write_cache_entry(cls, stream, entry):
 		""" Write an IndexEntry to a stream """
 		beginoffset = stream.tell()
-		stream.write(entry[4])			# ctime
-		stream.write(entry[5])			# mtime
+		write = stream.write
+		write(entry[4])			# ctime
+		write(entry[5])			# mtime
 		path = entry[3]
-		plen = len(path) & 0x0fff		# path length
+		plen = len(path) & CE_NAMEMASK		# path length
 		assert plen == len(path), "Path %s too long to fit into index" % entry[3]
-		flags = plen | (entry[2] << 12)# stage and path length are 2 byte flags
-		stream.write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0],
+		flags = plen | entry[2]
+		write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0],
 									entry[8], entry[9], entry[10], binascii.unhexlify(entry[1]), flags))
-		stream.write(path)
+		write(path)
 		real_size = ((stream.tell() - beginoffset + 8) & ~7)
-		stream.write("\0" * ((beginoffset + real_size) - stream.tell()))
+		write("\0" * ((beginoffset + real_size) - stream.tell()))
 
 	def write(self, file_path = None, ignore_tree_extension_data=False):
 		"""
@@ -272,7 +287,7 @@ def write(self, file_path = None, ignore_tree_extension_data=False):
 		if file_path is not None:
 			self._file_path = file_path
 
-	@clear_cache
+	@post_clear_cache
 	@default_index
 	def merge_tree(self, rhs, base=None):
 		"""Merge the given rhs treeish into the current index, possibly taking
@@ -383,24 +398,14 @@ def from_tree(cls, repo, *treeish, **kwargs):
 		return index
 
 	@classmethod
-	def _index_mode_to_tree_index_mode(cls, index_mode):
-		"""
-		Cleanup a index_mode value.
-		This will return a index_mode that can be stored in a tree object.
-
-		``index_mode``
-			Index_mode to clean up.
-		"""
-		if stat.S_ISLNK(index_mode):
-			return stat.S_IFLNK
-		elif stat.S_ISDIR(index_mode):
-			return stat.S_IFDIR
-		elif stat.S_IFMT(index_mode) == cls.S_IFGITLINK:
+	def _stat_mode_to_index_mode(cls, mode):
+		"""Convert the given mode from a stat call to the corresponding index mode
+		and return it"""
+		if S_ISLNK(mode):	# symlinks
+			return S_IFLNK
+		if S_ISDIR(mode) or S_IFMT(mode) == cls.S_IFGITLINK:	# submodules
 			return cls.S_IFGITLINK
-		ret = stat.S_IFREG | 0644
-		ret |= (index_mode & 0111)
-		return ret
-
+		return S_IFREG | 644 | (mode & 0100) 		# blobs with or without executable bit
 
 	# UTILITIES
 	def _iter_expand_paths(self, paths):
@@ -479,7 +484,9 @@ def iter_blobs(self, predicate = lambda t: True):
 			only if they match a given list of paths.
 		"""
 		for entry in self.entries.itervalues():
-			mode = self._index_mode_to_tree_index_mode(entry.mode)
+			# TODO: is it necessary to convert the mode ? We did that when adding 
+			# it to the index, right ?
+			mode = self._stat_mode_to_index_mode(entry.mode)
 			blob = Blob(self.repo, entry.sha, mode, entry.path)
 			blob.size = entry.size
 			output = (entry.stage, blob)
@@ -636,7 +643,6 @@ def _preprocess_add_items(self, items):
 		# END for each item
 		return (paths, entries)
 
-	@clear_cache
 	@default_index
 	def add(self, items, force=True, fprogress=lambda *args: None, path_rewriter=None):
 		"""Add files from the working tree, specific blobs or BaseIndexEntries
@@ -739,7 +745,7 @@ def store_path(filepath):
 			"""Store file at filepath in the database and return the base index entry"""
 			st = os.lstat(filepath)		# handles non-symlinks as well
 			stream = None
-			if stat.S_ISLNK(st.st_mode):
+			if S_ISLNK(st.st_mode):
 				stream = StringIO(os.readlink(filepath))
 			else:
 				stream = open(filepath, 'rb')
@@ -759,13 +765,6 @@ def store_path(filepath):
 			for filepath in self._iter_expand_paths(paths):
 				entries_added.append(store_path(filepath))
 			# END for each filepath
-			
-			# add the new entries to this instance, and write it
-			for entry in entries_added:
-				self.entries[(entry.path, 0)] = IndexEntry.from_base(entry)
-
-			# finally write the changed index
-			self.write()
 		# END path handling
 
 
@@ -823,6 +822,14 @@ def store_path(filepath):
 			self._flush_stdin_and_wait(proc, ignore_stdout=True)
 			entries_added.extend(entries)
 		# END if there are base entries
+
+		# FINALIZE
+		# add the new entries to this instance, and write it
+		for entry in entries_added:
+			self.entries[(entry.path, 0)] = IndexEntry.from_base(entry)
+
+		# finally write the changed index
+		self.write()
 
 		return entries_added
 
@@ -840,7 +847,7 @@ def _items_to_rela_paths(self, items):
 		# END for each item
 		return paths
 
-	@clear_cache
+	@post_clear_cache
 	@default_index
 	def remove(self, items, working_tree=False, **kwargs):
 		"""
@@ -893,7 +900,7 @@ def remove(self, items, working_tree=False, **kwargs):
 		# rm 'path'
 		return [ p[4:-1] for p in removed_paths ]
 
-	@clear_cache
+	@post_clear_cache
 	@default_index
 	def move(self, items, skip_errors=False, **kwargs):
 		"""
@@ -1127,7 +1134,7 @@ def handle_stderr(proc, iter_checked_out_files):
 		# END paths handling
 		assert "Should not reach this point"
 
-	@clear_cache
+	@post_clear_cache
 	@default_index
 	def reset(self, commit='HEAD', working_tree=False, paths=None, head=False, **kwargs):
 		"""

lib/git/index/typ.py

@@ -7,6 +7,15 @@
 
 __all__ = ('BlobFilter', 'BaseIndexEntry', 'IndexEntry')
 
+#{ Invariants
+CE_NAMEMASK = 0x0fff
+CE_STAGEMASK = 0x3000
+CE_EXTENDED = 0x4000
+CE_VALID = 0x8000
+CE_STAGESHIFT = 12
+
+#} END invariants
+
 class BlobFilter(object):
 	"""
 	Predicate to be used by iter_blobs allowing to filter only return blobs which
@@ -64,17 +73,22 @@ def stage(self):
 		
 		:note: For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html
 		"""
-		return self[2]
+		return (self[2] & CE_STAGEMASK) >> CE_STAGESHIFT
 
 	@property
 	def path(self):
 		""":return: our path relative to the repository working tree root"""
 		return self[3]
 
+	@property
+	def flags(self):
+		""":return: flags stored with this entry"""
+		return self[2]
+
 	@classmethod
 	def from_blob(cls, blob, stage = 0):
 		""":return: Fully equipped BaseIndexEntry at the given stage"""
-		return cls((blob.mode, blob.sha, stage, blob.path))
+		return cls((blob.mode, blob.sha, stage << CE_STAGESHIFT, blob.path))
 
 
 class IndexEntry(BaseIndexEntry):
@@ -121,22 +135,24 @@ def gid(self):
 	def size(self):
 		""":return: Uncompressed size of the blob """
 		return self[10]
-
+		
 	@classmethod
-	def from_base(cls, base):
+	def from_base(cls, base, actual_value=False):
 		""" 
 		:return:
 			Minimal entry as created from the given BaseIndexEntry instance.
 			Missing values will be set to null-like values
 
-		:param base: Instance of type BaseIndexEntry"""
+		:param base: Instance of type BaseIndexEntry
+		:param actual_value: if set, instead of writing 0 values for fields that 
+			don't exist in the BaseEntry, the actual values will be written in."""
 		time = pack(">LL", 0, 0)
-		return IndexEntry((base.mode, base.sha, base.stage, base.path, time, time, 0, 0, 0, 0, 0))
+		return IndexEntry((base.mode, base.sha, base.flags, base.path, time, time, 0, 0, 0, 0, 0))
 
 	@classmethod
-	def from_blob(cls, blob):
-		""":return: Minimal entry resembling the given blob objecft"""
+	def from_blob(cls, blob, stage = 0):
+		""":return: Minimal entry resembling the given blob object"""
 		time = pack(">LL", 0, 0)
-		return IndexEntry((blob.mode, blob.sha, 0, blob.path, time, time, 0, 0, 0, 0, blob.size))
+		return IndexEntry((blob.mode, blob.sha, stage << CE_STAGESHIFT, blob.path, time, time, 0, 0, 0, 0, blob.size))
 
 

lib/git/index/util.py

@@ -3,7 +3,7 @@
 import tempfile
 import os
 
-__all__ = ( 'TemporaryFileSwap', 'clear_cache', 'default_index' )
+__all__ = ( 'TemporaryFileSwap', 'post_clear_cache', 'default_index' )
 
 #{ Aliases 
 pack = struct.pack
@@ -36,7 +36,7 @@ def __del__(self):
 
 #{ Decorators 
 
-def clear_cache(func):
+def post_clear_cache(func):
 	"""Decorator for functions that alter the index using the git command. This would
 	invalidate our possibly existing entries dictionary which is why it must be
 	deleted to allow it to be lazily reread later.
@@ -45,14 +45,14 @@ def clear_cache(func):
 		This decorator will not be required once all functions are implemented
 		natively which in fact is possible, but probably not feasible performance wise.
 	"""
-	def clear_cache_if_not_raised(self, *args, **kwargs):
+	def post_clear_cache_if_not_raised(self, *args, **kwargs):
 		rval = func(self, *args, **kwargs)
 		self._delete_entries_cache()
 		return rval
 
 	# END wrapper method
-	clear_cache_if_not_raised.__name__ = func.__name__
-	return clear_cache_if_not_raised
+	post_clear_cache_if_not_raised.__name__ = func.__name__
+	return post_clear_cache_if_not_raised
 
 def default_index(func):
 	"""Decorator assuring the wrapped method may only run if we are the default

lib/git/objects/tree.py

@@ -121,7 +121,7 @@ class Tree(base.IndexObject, diff.Diffable, utils.Traversable, utils.Serializabl
 	__slots__ = "_cache"
 
 	# actual integer ids for comparison 
-	commit_id = 016
+	commit_id = 016		# equals stat.S_IFDIR | stat.S_IFLNK - a directory link
 	blob_id = 010
 	symlink_id = 012
 	tree_id = 004