Memcache item timestamps (#323)

* Add memcache item timestamps.

* v1 require google.golang.org/protobuf.

* Memcache new fields: use proto getters.

go.mod

@@ -5,4 +5,5 @@ go 1.11
 require (
 	github.com/golang/protobuf v1.5.2
 	golang.org/x/text v0.3.8
+	google.golang.org/protobuf v1.26.0
 )

internal/memcache/memcache_service.proto

@@ -27,6 +27,13 @@ message MemcacheGetRequest {
   optional string name_space = 2 [default = ""];
   optional bool for_cas = 4;
   optional AppOverride override = 5;
+  optional bool for_peek = 6 [default = false];
+}
+
+message ItemTimestamps {
+  optional int64 expiration_time_sec = 1;
+  optional int64 last_access_time_sec = 2;
+  optional int64 delete_lock_time_sec = 3;
 }
 
 message MemcacheGetResponse {
@@ -36,6 +43,8 @@ message MemcacheGetResponse {
     optional fixed32 flags = 4;
     optional fixed64 cas_id = 5;
     optional int32 expires_in_seconds = 6;
+    optional ItemTimestamps timestamps = 8;
+    optional bool is_delete_locked = 9;
   }
 }
 

memcache/memcache.go

@@ -81,6 +81,21 @@ type Item struct {
 	// casID is a client-opaque value used for compare-and-swap operations.
 	// Zero means that compare-and-swap is not used.
 	casID uint64
+
+	// ItemTimestamps are server values only returned when calling Peek and PeekMulti.
+	// The timestamps are nil when calling Get and GetMulti.
+	Timestamps ItemTimestamps
+}
+
+// ItemTimestamps are timestamps optionally provided by the server.
+// See Peek and PeekMulti.
+type ItemTimestamps struct {
+	// Expiration is related to Item.Expiration but it is a Time (not a Duration),
+	// provided by the server. It is not meant to be set by the user.
+	Expiration *time.Time
+	// LastAccess is the last time the Item was accessed.
+	LastAccess *time.Time
+	// The proto also includes delete_lock_time_sec, which is ignored in the Go lib.
 }
 
 const (
@@ -90,14 +105,34 @@ const (
 
 // protoToItem converts a protocol buffer item to a Go struct.
 func protoToItem(p *pb.MemcacheGetResponse_Item) *Item {
+	if p.GetIsDeleteLocked() {
+		// "delete lock" for a duration is not a feature available in the Go lib.
+		// Such items may exist in memcache though, e.g. created by the Java lib.
+		// In this case, nil is more appropriate than an item with empty value.
+		// For a single Get, nil will translate to ErrCacheMiss.
+		return nil
+	}
 	return &Item{
 		Key:   string(p.Key),
 		Value: p.Value,
 		Flags: p.GetFlags(),
 		casID: p.GetCasId(),
+		Timestamps: ItemTimestamps{
+			Expiration: timeSecToTime(p.GetTimestamps().GetExpirationTimeSec()),
+			LastAccess: timeSecToTime(p.GetTimestamps().GetLastAccessTimeSec()),
+		},
 	}
 }
 
+// For convenience, we interpret a 0 unix second timestamp as a nil *Time.
+func timeSecToTime(s int64) *time.Time {
+	if s == 0 {
+		return nil
+	}
+	t := time.Unix(s, 0)
+	return &t
+}
+
 // If err is an appengine.MultiError, return its first element. Otherwise, return err.
 func singleError(err error) error {
 	if me, ok := err.(appengine.MultiError); ok {
@@ -109,7 +144,35 @@ func singleError(err error) error {
 // Get gets the item for the given key. ErrCacheMiss is returned for a memcache
 // cache miss. The key must be at most 250 bytes in length.
 func Get(c context.Context, key string) (*Item, error) {
-	m, err := GetMulti(c, []string{key})
+	cas, peek := true, false
+	return get(c, key, cas, peek)
+}
+
+// GetMulti is a batch version of Get. The returned map from keys to items may
+// have fewer elements than the input slice, due to memcache cache misses.
+// Each key must be at most 250 bytes in length.
+func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
+	cas, peek := true, false
+	return getMulti(c, key, cas, peek)
+}
+
+// Peek gets the item for the given key and additionally populates Item.Timestamps.
+// ErrCacheMiss is returned for a memcache cache miss. The key must be at most 250
+// bytes in length.
+func Peek(c context.Context, key string) (*Item, error) {
+	cas, peek := true, true
+	return get(c, key, cas, peek)
+}
+
+// PeekMulti is a batch version of Peek. It is similar to GetMulti but
+// additionally populates Item.Timestamps.
+func PeekMulti(c context.Context, key []string) (map[string]*Item, error) {
+	cas, peek := true, true
+	return getMulti(c, key, cas, peek)
+}
+
+func get(c context.Context, key string, forCas bool, forPeek bool) (*Item, error) {
+	m, err := getMulti(c, []string{key}, forCas, forPeek)
 	if err != nil {
 		return nil, err
 	}
@@ -119,10 +182,7 @@ func Get(c context.Context, key string) (*Item, error) {
 	return m[key], nil
 }
 
-// GetMulti is a batch version of Get. The returned map from keys to items may
-// have fewer elements than the input slice, due to memcache cache misses.
-// Each key must be at most 250 bytes in length.
-func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
+func getMulti(c context.Context, key []string, forCas bool, forPeek bool) (map[string]*Item, error) {
 	if len(key) == 0 {
 		return nil, nil
 	}
@@ -131,8 +191,9 @@ func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
 		keyAsBytes[i] = []byte(k)
 	}
 	req := &pb.MemcacheGetRequest{
-		Key:    keyAsBytes,
-		ForCas: proto.Bool(true),
+		Key:     keyAsBytes,
+		ForCas:  proto.Bool(forCas),
+		ForPeek: proto.Bool(forPeek),
 	}
 	res := &pb.MemcacheGetResponse{}
 	if err := internal.Call(c, "memcache", "Get", req, res); err != nil {
@@ -141,7 +202,9 @@ func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
 	m := make(map[string]*Item, len(res.Item))
 	for _, p := range res.Item {
 		t := protoToItem(p)
-		m[t.Key] = t
+		if t != nil {
+			m[t.Key] = t
+		}
 	}
 	return m, nil
 }

memcache/memcache_test.go

@@ -261,3 +261,52 @@ func TestGetMultiEmpty(t *testing.T) {
 		t.Error("Service was called but should not have been")
 	}
 }
+
+func TestPeekRequest(t *testing.T) {
+	serviceCalled := false
+	key := "lyric"
+	value := "Where the buffalo roam"
+	t1, t2 := int64(1690570000), int64(1690564666)
+
+	c := aetesting.FakeSingleContext(t, "memcache", "Get", func(req *pb.MemcacheGetRequest, res *pb.MemcacheGetResponse) error {
+		// Test request.
+		if n := len(req.Key); n != 1 {
+			t.Errorf("got %d want 1", n)
+			return nil
+		}
+		if k := string(req.Key[0]); k != key {
+			t.Errorf("got %q want %q", k, key)
+		}
+
+		// Response
+		res.Item = []*pb.MemcacheGetResponse_Item{
+			{
+				Key:   []byte(key),
+				Value: []byte(value),
+				Timestamps: &pb.ItemTimestamps{
+					ExpirationTimeSec: &t1,
+					LastAccessTimeSec: &t2,
+				},
+			},
+		}
+		serviceCalled = true
+		return nil
+	})
+
+	item, err := Peek(c, key)
+	if !serviceCalled {
+		t.Error("Service was not called as expected")
+	}
+	if err != nil {
+		t.Errorf("got %v want nil", err)
+	}
+	if item == nil || item.Key != key || string(item.Value) != value {
+		t.Errorf("got %q, %q want {%q,%q}, nil", item, err, key, value)
+	}
+	if item.Timestamps.Expiration.Unix() != t1 {
+		t.Errorf("got %d, want %d", item.Timestamps.Expiration.Unix(), t1)
+	}
+	if item.Timestamps.LastAccess.Unix() != t2 {
+		t.Errorf("got %d, want %d", item.Timestamps.LastAccess.Unix(), t2)
+	}
+}

v2/internal/memcache/memcache_service.proto

@@ -27,6 +27,13 @@ message MemcacheGetRequest {
   optional string name_space = 2 [default = ""];
   optional bool for_cas = 4;
   optional AppOverride override = 5;
+  optional bool for_peek = 6 [default = false];
+}
+
+message ItemTimestamps {
+  optional int64 expiration_time_sec = 1;
+  optional int64 last_access_time_sec = 2;
+  optional int64 delete_lock_time_sec = 3;
 }
 
 message MemcacheGetResponse {
@@ -36,6 +43,8 @@ message MemcacheGetResponse {
     optional fixed32 flags = 4;
     optional fixed64 cas_id = 5;
     optional int32 expires_in_seconds = 6;
+    optional ItemTimestamps timestamps = 8;
+    optional bool is_delete_locked = 9;
   }
 }
 

v2/memcache/memcache.go

@@ -81,6 +81,21 @@ type Item struct {
 	// casID is a client-opaque value used for compare-and-swap operations.
 	// Zero means that compare-and-swap is not used.
 	casID uint64
+
+	// ItemTimestamps are server values only returned when calling Peek and PeekMulti.
+	// The timestamps are nil when calling Get and GetMulti.
+	Timestamps ItemTimestamps
+}
+
+// ItemTimestamps are timestamps optionally provided by the server.
+// See Peek and PeekMulti.
+type ItemTimestamps struct {
+	// Expiration is related to Item.Expiration but it is a Time (not a Duration),
+	// provided by the server. It is not meant to be set by the user.
+	Expiration *time.Time
+	// LastAccess is the last time the Item was accessed.
+	LastAccess *time.Time
+	// The proto also includes delete_lock_time_sec, which is ignored in the Go lib.
 }
 
 const (
@@ -90,14 +105,34 @@ const (
 
 // protoToItem converts a protocol buffer item to a Go struct.
 func protoToItem(p *pb.MemcacheGetResponse_Item) *Item {
+	if p.GetIsDeleteLocked() {
+		// "delete lock" for a duration is not a feature available in the Go lib.
+		// Such items may exist in memcache though, e.g. created by the Java lib.
+		// In this case, nil is more appropriate than an item with empty value.
+		// For a single Get, nil will translate to ErrCacheMiss.
+		return nil
+	}
 	return &Item{
 		Key:   string(p.Key),
 		Value: p.Value,
 		Flags: p.GetFlags(),
 		casID: p.GetCasId(),
+		Timestamps: ItemTimestamps{
+			Expiration: timeSecToTime(p.GetTimestamps().GetExpirationTimeSec()),
+			LastAccess: timeSecToTime(p.GetTimestamps().GetLastAccessTimeSec()),
+		},
 	}
 }
 
+// For convenience, we interpret a 0 unix second timestamp as a nil *Time.
+func timeSecToTime(s int64) *time.Time {
+	if s == 0 {
+		return nil
+	}
+	t := time.Unix(s, 0)
+	return &t
+}
+
 // If err is an appengine.MultiError, return its first element. Otherwise, return err.
 func singleError(err error) error {
 	if me, ok := err.(appengine.MultiError); ok {
@@ -109,7 +144,35 @@ func singleError(err error) error {
 // Get gets the item for the given key. ErrCacheMiss is returned for a memcache
 // cache miss. The key must be at most 250 bytes in length.
 func Get(c context.Context, key string) (*Item, error) {
-	m, err := GetMulti(c, []string{key})
+	cas, peek := true, false
+	return get(c, key, cas, peek)
+}
+
+// GetMulti is a batch version of Get. The returned map from keys to items may
+// have fewer elements than the input slice, due to memcache cache misses.
+// Each key must be at most 250 bytes in length.
+func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
+	cas, peek := true, false
+	return getMulti(c, key, cas, peek)
+}
+
+// Peek gets the item for the given key and additionally populates Item.Timestamps.
+// ErrCacheMiss is returned for a memcache cache miss. The key must be at most 250
+// bytes in length.
+func Peek(c context.Context, key string) (*Item, error) {
+	cas, peek := true, true
+	return get(c, key, cas, peek)
+}
+
+// PeekMulti is a batch version of Peek. It is similar to GetMulti but
+// additionally populates Item.Timestamps.
+func PeekMulti(c context.Context, key []string) (map[string]*Item, error) {
+	cas, peek := true, true
+	return getMulti(c, key, cas, peek)
+}
+
+func get(c context.Context, key string, forCas bool, forPeek bool) (*Item, error) {
+	m, err := getMulti(c, []string{key}, forCas, forPeek)
 	if err != nil {
 		return nil, err
 	}
@@ -119,10 +182,7 @@ func Get(c context.Context, key string) (*Item, error) {
 	return m[key], nil
 }
 
-// GetMulti is a batch version of Get. The returned map from keys to items may
-// have fewer elements than the input slice, due to memcache cache misses.
-// Each key must be at most 250 bytes in length.
-func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
+func getMulti(c context.Context, key []string, forCas bool, forPeek bool) (map[string]*Item, error) {
 	if len(key) == 0 {
 		return nil, nil
 	}
@@ -131,8 +191,9 @@ func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
 		keyAsBytes[i] = []byte(k)
 	}
 	req := &pb.MemcacheGetRequest{
-		Key:    keyAsBytes,
-		ForCas: proto.Bool(true),
+		Key:     keyAsBytes,
+		ForCas:  proto.Bool(forCas),
+		ForPeek: proto.Bool(forPeek),
 	}
 	res := &pb.MemcacheGetResponse{}
 	if err := internal.Call(c, "memcache", "Get", req, res); err != nil {
@@ -141,7 +202,9 @@ func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
 	m := make(map[string]*Item, len(res.Item))
 	for _, p := range res.Item {
 		t := protoToItem(p)
-		m[t.Key] = t
+		if t != nil {
+			m[t.Key] = t
+		}
 	}
 	return m, nil
 }

v2/memcache/memcache_test.go

@@ -261,3 +261,52 @@ func TestGetMultiEmpty(t *testing.T) {
 		t.Error("Service was called but should not have been")
 	}
 }
+
+func TestPeekRequest(t *testing.T) {
+	serviceCalled := false
+	key := "lyric"
+	value := "Where the buffalo roam"
+	t1, t2 := int64(1690570000), int64(1690564666)
+
+	c := aetesting.FakeSingleContext(t, "memcache", "Get", func(req *pb.MemcacheGetRequest, res *pb.MemcacheGetResponse) error {
+		// Test request.
+		if n := len(req.Key); n != 1 {
+			t.Errorf("got %d want 1", n)
+			return nil
+		}
+		if k := string(req.Key[0]); k != key {
+			t.Errorf("got %q want %q", k, key)
+		}
+
+		// Response
+		res.Item = []*pb.MemcacheGetResponse_Item{
+			{
+				Key:   []byte(key),
+				Value: []byte(value),
+				Timestamps: &pb.ItemTimestamps{
+					ExpirationTimeSec: &t1,
+					LastAccessTimeSec: &t2,
+				},
+			},
+		}
+		serviceCalled = true
+		return nil
+	})
+
+	item, err := Peek(c, key)
+	if !serviceCalled {
+		t.Error("Service was not called as expected")
+	}
+	if err != nil {
+		t.Errorf("got %v want nil", err)
+	}
+	if item == nil || item.Key != key || string(item.Value) != value {
+		t.Errorf("got %q, %q want {%q,%q}, nil", item, err, key, value)
+	}
+	if item.Timestamps.Expiration.Unix() != t1 {
+		t.Errorf("got %d, want %d", item.Timestamps.Expiration.Unix(), t1)
+	}
+	if item.Timestamps.LastAccess.Unix() != t2 {
+		t.Errorf("got %d, want %d", item.Timestamps.LastAccess.Unix(), t2)
+	}
+}