From brad@danga.com Thu Aug 7 17:53:30 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Thu, 7 Aug 2003 09:53:30 -0700 (PDT) Subject: summary of misc off-list traffic Message-ID: Uriah Welcome is looking at using memcached for SourceForge.net, so he started working on a Python API, but without multiple servers. Evan (at danga) then ported the Perl one to Python, and he's cleaning it up now. I'll put it up and announce it when it's ready. Somebody asked about porting memcached to Windows, but we don't see the point: paying for a Windows license for each box? Ouch! But, I guess if you're just sticking memcache on existing Windows web nodes, and you're already paying for the license, then it'd work. (that is recommended, btw: take your existing web nodes, add more memory, and also run memcached on them.... memcached takes almost no CPU.) In any case, if you want to run memcached on Windows, port it, then send us patches and we'll integrate them. Avva (works for Danga and wrote most of memcached server) is working on making object expirations happen as soon as possible, instead of falling off the LRU, or being killed on retrieve. Currently it's possible for an object with a lifetime of 30 seconds to live in memory for hours, as long as nobody accesses it. If somebody does access it 31 seconds later, or an hour later, its expiration time is noted and it's then destroyed, and the client gets an empty result, but it was still in memory longer than it needed to be, so it was probably pushing other objects with infinite expiration times off the LRU. So, the early expiration patch will increase hit rates a little. Probably a tiny amount, but it really depends on what you're throwing in memcache. Don't be afraid to use the list. Everybody I've talked to off-list has been smart, friendly, and had good questions/advice. - Brad From martine@danga.com Thu Aug 7 23:04:23 2003 From: martine@danga.com (Evan Martin) Date: Thu, 7 Aug 2003 15:04:23 -0700 Subject: preliminary python memcache client Message-ID: <20030807220423.GA15808@danga.com> --FCuugMFkClbJLl1L Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Attached is a patch against memcached CVS that adds a Python client. It currently lacks some features, but it should provide the basic functionality you'd need to try using memcached with Python. It also lacks docs, but it pretty much follows the Perl API; there's also a simple example included at the end of the module itself ('python memcached.py'). You'll need to be running a memcached process before the test does anything useful. Bug reports / feature requests / enhancements are welcome. Finally, it's been a while since I've used Python, so I'd appreciate stylistic feedback: am I using the proper idioms, am I naming/capitalizing things the way you'd expect, etc. If anyone here is knowledgable about those sorts of things, please respond to me off-list, as I have a lot of questions (like: which of the doc tools should I use?). -- Evan Martin martine@danga.com http://neugierig.org --FCuugMFkClbJLl1L Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="memcached-python.patch" diff -ruN memcache-orig/api/python/ChangeLog memcache-python/api/python/ChangeLog --- memcache-orig/api/python/ChangeLog 1969-12-31 16:00:00.000000000 -0800 +++ memcache-python/api/python/ChangeLog 2003-08-07 14:24:20.000000000 -0700 @@ -0,0 +1,4 @@ +Thu, 07 Aug 2003 14:20:27 -0700 Evan Martin + + * Initial prerelease. + diff -ruN memcache-orig/api/python/MANIFEST memcache-python/api/python/MANIFEST --- memcache-orig/api/python/MANIFEST 1969-12-31 16:00:00.000000000 -0800 +++ memcache-python/api/python/MANIFEST 2003-08-07 14:21:38.000000000 -0700 @@ -0,0 +1,3 @@ +memcache.py +setup.py +ChangeLog diff -ruN memcache-orig/api/python/memcache.py memcache-python/api/python/memcache.py --- memcache-orig/api/python/memcache.py 1969-12-31 16:00:00.000000000 -0800 +++ memcache-python/api/python/memcache.py 2003-08-07 15:02:21.000000000 -0700 @@ -0,0 +1,367 @@ +#!/usr/bin/env python + +"""MemCachedClient: a client for memcached. + +http://www.danga.com/memcached""" + +import sys +import socket +import pickle +import time +import re +import string +import types + +__author__ = "Evan Martin " +__version__ = "1.0" +__copyright__ = "Copyright (C) 2003 Danga Interactive" +__license__ = "Python" + +class Error(Exception): + pass + +class Client: + _valuere = re.compile(r'^VALUE (\S+) (\d+) (\d+)') + _FLAG_PICKLE = 1<<0 + _FLAG_INTEGER = 1<<1 + _FLAG_LONG = 1<<2 + + _SERVER_RETRIES = 10 # how many times to try finding a free server. + + def __init__(self, servers, debug=0): + self.set_servers(servers) + self.debug = debug + self.stats = {} + + def set_servers(self, servers): + self.servers = [Host(s, self.debuglog) for s in servers] + self.init_buckets() + + def debuglog(self, str): + if self.debug: + sys.stderr.write("MemCached: %s\n" % str) + + def statlog(self, func): + if not self.stats.has_key(func): + self.stats[func] = 1 + else: + self.stats[func] += 1 + + def forget_dead_hosts(self): + for s in self.servers: + s.dead_until = 0 + + def init_buckets(self): + self.buckets = [] + for server in self.servers: + for i in range(server.weight): + self.buckets.append(server) + + def get_server(self, key): + if type(key) == types.TupleType: + serverhash = key[0] + key = key[1] + else: + serverhash = hash(key) + + for i in range(Client._SERVER_RETRIES): + server = self.buckets[serverhash % len(self.buckets)] + if server.connect(): + #print "(using server %s)" % server, + return server + serverhash = hash(str(serverhash) + str(i)) + return None + + def disconnect_all(self): + for s in self.servers: + s.close_socket() + + def delete(self, key, time): + # XXX implement me. + pass + + def incr(self, key, time): + # XXX implement me. + pass + + def decr(self, key, time): + # XXX implement me. + pass + + def add(self, key, val, time=0): + '''Add {key: val}, expiring at time.''' + return self._set("add", key, val, time) + def replace(self, key, val, time=0): + '''Replace existing key with {key: val}, expiring at time.''' + return self._set("replace", key, val, time) + def set(self, key, val, time=0): + '''Set {key: val}, expiring at time.''' + return self._set("set", key, val, time) + + def _set(self, cmd, key, val, time): + server = self.get_server(key) + if not server: + return 0 + + self.statlog(cmd) + + flags = 0 + if isinstance(val, types.StringTypes): + pass + elif isinstance(val, int): + flags |= Client._FLAG_INTEGER + val = "%d" % val + elif isinstance(val, long): + flags |= Client._FLAG_LONG + val = "%d" % val + else: + flags |= Client._FLAG_PICKLE + val = pickle.dumps(val, 2) + + fullcmd = "%s %s %d %d %d\r\n%s" % (cmd, key, flags, time, len(val), val) + try: + server.send_cmd(fullcmd) + server.expect("STORED") + except socket.error, msg: + server.mark_dead(msg[1]) + return 0 + return 1 + + def get(self, key): + server = self.get_server(key) + if not server: + return None + + self.statlog('get') + + try: + server.send_cmd("get %s" % key) + rkey, flags, rlen, = self._expectvalue(server) + if not rkey: + return None + value = self._recv_value(server, flags, rlen) + server.expect("END") + except (Error, socket.error), msg: + if type(msg) is types.TupleType: + msg = msg[1] + server.mark_dead(msg) + return None + return value + + def get_multi(self, *keys): + self.statlog('get_multi') + + server_keys = {} + + # build up a list for each server of all the keys we want. + for key in keys: + server = self.get_server(key) + if not server: + continue + if type(key) == types.TupleType: + key = key[1] + if not server_keys.has_key(server): + server_keys[server] = [] + server_keys[server].append(key) + + # send out all requests on each server before reading anything + dead_servers = [] + for server in server_keys.keys(): + try: + server.send_cmd("get %s" % string.join(server_keys[server])) + except socket.error, msg: + server.mark_dead(msg[1]) + dead_servers.append(server) + + # if any servers died on the way, don't expect them to respond. + for server in dead_servers: + del server_keys[server] + + retvals = {} + for server in server_keys.keys(): + try: + line = server.readline() + while line and line != 'END': + rkey, flags, rlen = self._expectvalue(server, line) + val = self._recv_value(server, flags, rlen) + retvals[rkey] = val + line = server.readline() + except (Error, socket.error), msg: + server.mark_dead(msg) + return retvals + + def _expectvalue(self, server, line=None): + if not line: + line = server.readline() + match = self._valuere.match(line) + if not match: + return (None, None, None) + rkey = match.group(1) + flags = int(match.group(2)) + rlen = int(match.group(3)) + return (rkey, flags, rlen) + + def _recv_value(self, server, flags, rlen): + rlen += 2 # include \r\n + buf = server.recv(rlen) + if len(buf) != rlen: + raise Error("received %d bytes when expecting %d" % (len(buf), rlen)) + + if len(buf) == rlen: + buf = buf[:-2] # strip \r\n + + if flags == 0: + val = buf + elif flags & Client._FLAG_INTEGER: + val = int(buf) + elif flags & Client._FLAG_LONG: + val = long(buf) + elif flags & Client._FLAG_PICKLE: + val = pickle.loads(buf) + else: + self.debuglog("unknown flags on get: %x\n" % flags) + + return val + + +class Host: + """A single server in the memcachepool that MemCachedClient chooses from.""" + def __init__(self, host, debugfunc=None): + if isinstance(host, types.TupleType): + host = host[0] + self.weight = host[1] + else: + self.weight = 1 + + if string.find(host, ":") > 0: + self.ip, self.port = string.split(host, ":") + self.port = int(self.port) + else: + self.ip, self.port = host, 11211 + + if not debugfunc: + debugfunc = lambda x: x + self.debuglog = debugfunc + + self.deaduntil = 0 + self.socket = None + + def _check_dead(self): + if self.deaduntil and self.deaduntil > time.time(): + return 1 + self.deaduntil = 0 + return 0 + + def connect(self): + if self._get_socket(): + return 1 + return 0 + + def mark_dead(self, reason): + print "MemCache: %s: %s. Marking dead." % (self, reason) + self.deaduntil = time.time()+5 #XXX magic + self.close_socket() + + def _get_socket(self): + if self._check_dead(): + return None + if self.socket: + return self.socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + #s.settimeout(1) # XXX magic number + try: + s.connect((self.ip, self.port)) + except socket.error, msg: + self.mark_dead("connect: %s" % msg[1]) + return None + self.socket = s + return s + + def close_socket(self): + if self.socket: + self.socket.close() + self.socket = None + + def send_cmd(self, cmd): + self.socket.sendall(cmd + "\r\n") + + def readline(self): + newlines = 0 + buf = '' + while newlines < 2: + char = self.socket.recv(1) # XXX does this buffer or is this slow? + if len(char) == 0: + # connection closed. + print "MemCache: Connection closed while reading from %s. Marking dead." % self + self.mark_dead + return buf + if char == '\r' and newlines == 0: + newlines = 1 + elif char == '\n' and newlines == 1: + newlines = 2 + else: + newlines = 0 + buf = buf + char + return buf + + def expect(self, text): + line = self.readline() + if line != text: + self.debuglog("while expecting '%s', got unexpected response '%s'" % (text, line)) + return line + + def recv(self, rlen): + buf = '' + while len(buf) < rlen: + buf = buf + self.socket.recv(rlen - len(buf)) + return buf + + def __str__(self): + d = '' + if self.deaduntil: + d = " (dead until %d)" % self.deaduntil + return "%s:%d%s" % (self.ip, self.port, d) + +if __name__ == "__main__": + print "Running tests:" + print + #servers = ["127.0.0.1:11211", "127.0.0.1:11212"] + servers = ["127.0.0.1:11211"] + mc = Client(servers, debug=1) + + def to_s(val): + if not isinstance(val, types.StringTypes): + return "%s (%s)" % (val, type(val)) + return "%s" % val + def test_setget(key, val): + print "Testing set/get {'%s': %s} ..." % (to_s(key), to_s(val)), + mc.set(key, val) + newval = mc.get(key) + if newval == val: + print "OK" + else: + print "FAIL" + + class FooStruct: + def __init__(self): + self.bar = "baz" + def __str__(self): + return "A FooStruct" + def __eq__(self, other): + if isinstance(other, FooStruct): + return self.bar == other.bar + return 0 + + test_setget("a_string", "some random string") + test_setget("an_integer", 42) + print "Testing get_multi ...", + print mc.get_multi("a_string", "an_integer") + test_setget("long", long(1<<30)) + + print "Testing get(unknown value) ...", + print to_s(mc.get("unknown_value")) + + f = FooStruct() + test_setget("foostruct", f) + +# vim: ts=4 sw=4 et : diff -ruN memcache-orig/api/python/setup.py memcache-python/api/python/setup.py --- memcache-orig/api/python/setup.py 1969-12-31 16:00:00.000000000 -0800 +++ memcache-python/api/python/setup.py 2003-08-07 15:02:34.000000000 -0700 @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +from distutils.core import setup +import memcache + +setup(name="memcache", + version=memcache.__version__, + author="Evan Martin", + author_email="martine@danga.com", + url="http://www.danga.com/memcached", + py_modules=["memcache"]) + diff -ruN memcache-orig/api/python/tests/pooltest.py memcache-python/api/python/tests/pooltest.py --- memcache-orig/api/python/tests/pooltest.py 1969-12-31 16:00:00.000000000 -0800 +++ memcache-python/api/python/tests/pooltest.py 2003-08-07 14:26:38.000000000 -0700 @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +"""pooltest + +Bring up two memcaches on :11211 and :11212. Try killing one or both. +If this code raises any exceptions, it's a bug.""" + +import memcache +import time + +mc = memcache.Client(["127.0.0.1:11211", "127.0.0.1:11212"], debug=1) + +def test_setget(key, val): + print "Testing set/get {'%s': %s} ..." % (key, val), + mc.set(key, val) + newval = mc.get(key) + if newval == val: + print "OK" + else: + print "FAIL" + +i = 0 +while 1: + test_setget("foo%d" % i, "bar%d" % i) + time.sleep(1) + i += 1 diff -ruN memcache-orig/CONTRIBUTORS memcache-python/CONTRIBUTORS --- memcache-orig/CONTRIBUTORS 2003-07-28 01:40:02.000000000 -0700 +++ memcache-python/CONTRIBUTORS 2003-08-07 15:02:10.000000000 -0700 @@ -10,8 +10,9 @@ -- C server -- memory allocator design -Evan Martine +Evan Martin -- automake/autoconf support + -- Python client Ryan -- PHP client --FCuugMFkClbJLl1L-- From martine@danga.com Fri Aug 8 03:23:21 2003 From: martine@danga.com (Evan Martin) Date: Thu, 7 Aug 2003 19:23:21 -0700 Subject: python api, try two Message-ID: <20030808022321.GC17601@danga.com> --4Ckj6UjgE2iN1+kY Content-Type: text/plain; charset=us-ascii Content-Disposition: inline This is 1.1. * Add incr, decr, and delete. * Better Python (based on comments from Uriah Welcome). * Docs, using epydoc. Hopefully it's ready for CVS now. -- Evan Martin martine@danga.com http://neugierig.org --4Ckj6UjgE2iN1+kY Content-Type: application/octet-stream Content-Disposition: attachment; filename="python-memcache-1.1.tar.gz" Content-Transfer-Encoding: base64 H4sICEUJMz8CA3B5dGhvbi1tZW1jYWNoZS0xLjEudGFyAO08f3PbNrL59/FToMzkWWxlWpKd eIYT5dV13J6nbtJxnF5vUo+GJiGJNUXqCNKKzs/f/e0uQBCkKNluk3TendhGlghgsdjfuwA5 X+bTNNmd8VngB1O+23f7e08+8dXrHfQOnz+Hv73+4fOe+be8nvQOD/sHL3oHz3vQr3+wv997 wp4/+QJXIXI/Y+zJzM/yKOFr+93X/v/0mrfwP+ciF3t/Jf+fHxwOtvz/a/k/T9MYv7nz5Z/l f7+3if/93sGB5P/zg0Gv/wL6ww/gf2/L/89+Pf1qrxDZ3lWU7PHkhklxsCzbtkv+W9Z3WZRM WDFn+SJlpaQIlibM6/cH/T7zk1B+HbiMXWRLdh3FMY5JE87SjF2l+dS1Tscsn0aCBWnIWeZH AmD4yZLxjwGf51GaiC6L8h24ya6KiQsoWFY0m6dZrictf+fRjFvWLGBD3eQexxFP8s4Huz84 dHvwX19iZ3dZ49bAvuyykMMkw75jWSEHxGChI8HzCc8713zZZTd+7HgWg2sOq8+ZfQE9cEnQ aQ96sdudZ2LHY8/EHXNd12bPWDWwSwNngStMcHQz4Qv4jngHrppMNkRj3TbE7nJyA4G3P9p0 i8eCrzR+f3R6huQCwD1rMY1izvqyk7kye5ymz0LENQKiXPmZ+iExQKK6IuZ83ukrnNg3Q9a3 nmyvf9urzf7//OMPu6dvvn/7yfw/2P8XaOPX+P/9g1X/3z/c2v8vcf3Ecz/0c3/3F54JsMEe 67s9640/4x5ryIZldOlb74oZkGTpsfdvfnzz9u9vrL+lM7479ycwcJrnc29vb7FYuKGfTHw3 SGd7JZjQOioAbuaxkxs/YT8RXdW9XT7zo9hjitjf6tHWWRTwBAyfnu41F0EWkd+obv4c+/k4 zWbVna2GP17/wVsU8z8f9j1Y/yHbK/Uf/g2g/4ve/out/v+F8d84S2csjERe5FEsQAMzzlTs RdKxEphZdLuTgOEY2g2hsrsqXLmRBmSoY7bRSN0ajco+PhmCoW0YB7veNiIjMbRXjITuV2TQ vNkI6b7z5WiWhkXMxfCDrTG+hLjwP1b/j6dAL36WTr6U/mPOp/R/Hxw/6v/hVv+/yHUxLbqs d8iOigkbQCjG+i+8/QH8z3Z7h70eM500Yy9XdO6VZf3X1+woDFmUBBmmVfiJCWHIY55zF5u/ 43nOM/YzyRrrXPmCh5g+wvgZ5GyCkbl5n0X+lP2dx3CbOzTwdRpAUlgITLz4fBmmgWtZqygf eIOeNzh8FMqnSZRHkG3NM54BpoCSa/0HRgtt+q/N8ycKATbrP4T6zw90/H+wT/oPX7f6/9fW f6yACipMekcGYbX29iHrwNc0WzL6yUIffiXgMt+CO7+J+MIaqsuy3nHO3t/m0OsnPjuWo6eg 4JgpvNzso1/dyVkx+PCv0qIKN0LQ1fcCIDAh0xA9oZr1AgtNYpoWccgm0Q1ny7RgPhtzHhPM abqQxSi1unTOMz/nwvMsWflohDeyntNWbtpplJt2dG2p51hmHcgWsOrRNV9iQeodfMcqT8Ft WWqh71VZqOrcAOInKdAyU3D2nbJRWtt6u2ysDVfT920Hbj5l0JkDHfycIYOgEWwt8AcphOZ8 D605m4GKsCtgARM5VgLdEiJ2kSANNKpbwATgTw6+wM9CtvCXLE8RvqYhW0T5FMBi/ok+gQE/ 4uiaE2dKTiBSQ6BoBlzEVXXSq9/ldPBlTR0tzbGxKpLJnld+cM2TcOTPIxriuq6juygCEXC6 +ZQtOEBasKkP4gP3pVcbF3kB4jj3BZYv82mWFpNpVdcsh0ZxTEtFssJYHuTSx+FvKT8WZLA5 xLFAbvByBfpBH5PZuiQbIv0TqkFYjgnNMUg4/wYa/CuQZfB6OM3ZrRTROxbEgK1rmQVVsRT6 awpkyWuV1fL7cs6FlUOSb+pE8HMUXMM0vmBz+mbJAi47pfaTLIPc3hygelmjkYrfR9g4ZGaI 3+ambctID+QI8E54N0jnyyyaTHO4D3ePy5+sc+zIkOA1QgEfD2GHH+QgOjgsljWEEpgMR4Am RB82Isw7J2U1uiz/QlvZRRJU3kdq4t+3krsZh0BCQCuGKj7D4jlLx5WoCzCMsBa3Ukk0jGe3 ZQdp6oAeqTKhrhTD04T5IEuBj+K2AM1GPUSdAJajrna15gYwGLSUR6j/lez3XXbEBLACWDb1 xZREBDnLOlKduyAwOZ9wCNt4HriOq4cOcChkdWgfx+z4toPjyVB1cUbnDqv9KPkSl3ERo/ah pV344DhA2/2bNAorHfOvkTqm3Q38OChisLywKoQu7SDA/QdAmYHJAKqOETWgjobDP/q4nC7O cM35nCgEGPpk6hPEJdsRSu9onwJpJCA1rRt0vCRfuqAFhHhAHqPUXAWoSKJ/gnGOQpR5vG9g Krn07QTswBxYCtTy2GgUQXAJKS3myiPFeVoCmJ1RyP1wNE1FDrcgww7SJAE0R7AG5TjidGIC PQWRzWSZC8CBFQqB6SBusR9ws985B3Zy3DlgE+yHc82KOI/qwIjVwjPC9TqQWUogpD+RTQK0 +FOsqvubJjtRvrGYbgPvbi2lMBAqVW/0/dnRD6OfT49/PDtBje6/fNkzGk7fXJz8cHIuG/pG w9nbNz9IEwQNA8nB0buT819OzkfnJxfnpyfvsK2HlhwDhRnuUqFhFChxYA/ZOEpCqejjjJfK rWQBt5NKWnUEj8ddpolVBgaVepZrwes441ITEr5QtqZ0HuQpUfSkhMcRuGSQeG1VNIxv537m z8oGD7RWfb1DLSW3FeIqzm4NLt65zfGEqIcGB60JDgCmApeWjKOVBD0SGH6RSUowMiBAaIN2 MFbQ4EAOcjDAGLC1rRjJ4xqIdNRfp96DsKEoAP42BoMHFNB0e1dZVuRAHajBhDW0B9Ul+paG W/WW8dDVUnl44skGaoMJ9rMMqGTypppCglSGWvECvfUiRTWaCc/UD7Tc78hCCwSHyGEn4KiN euahb7XvusCBKJiirwXkcO8UVu+D+kD4Qi4RhvbdGlww6xdo1JtgOwbcrhru0ATodY5v5R2U oxo4WI1yIOWUyjJuYLgkxJB9GP0NpuyIrsFnMBQOOUOBtFF9L+sQpHpdFRi5iI5TKV4JoeR5 nhn8jsbGNHVaQzwEogRRZuYusggDaZ2t4B7vbwnukyI0Q8dR9KqpxkUS1OfCMLSSURecBsWv jY51Sf6ArZdofSpvV9vvbe1Ou7QasRWLTBiukftzCFpyxoHGS4ady+iR9AA9eMJsP8YACkP5 fA1bDXZV/G0g7RJCBURIMe1SV4SsMbOBKwFUbSgwl/U5pdnZODElM9glw6qmsjCuEu961+aE rj+fQ8agxpi8n2jzotiP8VCN+xhiUVaC+/kUSbukdRfwtclOhEMxxRDhfOhd1tplBoQN/cuN UmGAwT8yJ7LW0EH6F7fh95w2mADPpMoHY6ZnLOZJx2x16shHpSF0VUjQaaH5U3WKoSMrfWrW Z8IhrZMR2sqgDIKtLNEBHCz2HnKA+naquw77BhW6EzmVs1Eg36QJBCP4aRiWWkzTlNIHyX8Q p5B8yHSr49TdlUrftSx1Kd6oRQo7OzuvqZdQGYBOKHVNQvetHJRckoer+RfPUgyGRREEXBiO 6dsM5RNDwtycbSVGloIoDbChADr1rtk9bKpTQFG31zDlpRndkTTYqcFCKrCvhsSMOrRgFmIS JweBrDB5lqWjieds0JXmYByJAqQ76aS3TgNgbRKOYHQH/jltHfjHOQq5/frk7OTi5LVtYCGz ZMl/l8KoLpuJSes8kA1fkwXvQA/Qe2czJdUNwwlQbcYQJ1hq7g/7a6OfJESxwoI8VjkwzNVJ K7mBPJ1FkKzFS4LMsfRAXWTdCuX/+BYmuoNYqYpnb2lWCCHwDNiS9ZGj5U1KGhMB1IrGEcSH 6IpwEYJ4LXsSwDDlAqJKg44U/FaaTwWvRYT1o1xRQqZpGEZL/PxxzrMKcypiaYBvakWwlQVR +cuPITYPl2py5SNLvZOVoahCsayYEQ0phKvqA7Jgg9mqjpsMXF69eqVrdZCLYv0C63WDHhbs nurV9TE3CoEh8kiaVOgq5W+Ck4W6El4lS4OH9xxUOGKRdxxDXqR5gLxErQdagHAjL78Dmi/8 jAoHRFGQrvk8S4FcgHEF9uuv9/HMoKyFYKJ3txpdk7x4ZerK/Blih0JZSSIIV0dVe4Hs0tBl bEL5VOa4LeZwo2hsMoym2mgXJEPSIJPlT/xmm3pXC1Afo5dnWA09u0WAoEVXRU7jCVEBZHuf xLUORRIq1mCyIpmBsqnhaY0AdmRYfYH4BnrkrAfQTktXrSgTkZz51ZTIzEFj8YL1usT73f4j OFeB/NOc06A+Befwcx3nqm6SfeABaj29L+wvax5IOTR0g9oTNhH8E84tjhJOS6F+aArxTqfV KwHJO9j6OT1fPTbzw9BUKZCO9vBJf8ftalQFZBBVVhoZa1P/wBwr9UsT8IAiT8GWoxfQ5d1o rOuwyl/VXYapG58iJKvJr9wYCkO7SQBHk0iV2B5EpnPZVyKO/uWzkKlGHjRmGhpuG6XzeSog D0eTc3YLa7v7/BRUNNpARez2EAq+TyBXCCP09BQyCUxeZdCOsZQq4ikrmzRi+BodVAyCJaN0 ruHhZlwiK/PdqjQ4jjIIOyCklYaVy1p7VYurqtbd1jr9I+rzGmZLmf6PlecrI2eW6f94eV7D M8v0n1F44GON4BgFIy0+lWmuOn8+/7HJgVQpbOxPBBVmDNAR5OO4gwuWQyJKdQxZlcRChmgk 87RZVqVdLRCAoo0xcuL/HbKyImEW72s95YMTtnRv8GPzTHGaTB40FW4HPHSeZiLZDlFuSbTA lFuhbljM5kJiOXBWzem4iOOGQ5f//5b9lshk1XDuhIKUoS5VY+gJlOqJkwd5fDXlxpT23cXb 878wo52YxrfpsWjr649VRi500ge2CwML9wsUQaoApqUQAmB3nEfUImx8HkkXMRqENWUkA/Ho auwlX2npusLZKN3hKhCCt676RutoyHnB9RQZD25G5gw1VDZK28mbFlHryE36bk3mnBahKwuw 0OBgIrOxAEspu8ANJima94rv+pi0cY+WXhNhubVZCbJYE6JWIk0DcA8ee6+KNsSb6LR/L6go whm4wmzZIu+Y4Cs/J0/NCPVAmC0fBTOI3d71yv8XdD0YOM2aQbWqDxU87Iu/EPIl1cBvqdGT k6k+HsC7M+oWRuyDMQjPp2mI7AM5oqOiIZ4bxQw14xMISzKIDWH2O3T8UQ4Wf4HbSlT6KWZX 0C0dV1qU5nja05c7CeN4Sfu3GWSkIYYWGSQE+SLNrnE7OiwCCoRoCEY/SQBKBP4lqDhMYyB/ 1vE+nROCkGfhAyoUBwF7GE2wC7ycy1CW0L3iYzzMgwpsVkNkzeoj8XE1mUb2e+zI2GHEOy0Z MjtiYRRQsJiV/fakXs79iAjk4/YgHiUpDw0Z06EUbrBMktWmfZIaMiLxpI1Y3fIUkoEIAzaI AeWetaaLqjRgWAjRoUoKBJ65wpjUrVXW6bhLIgnQopsPs8n32OVyszpKCr5+DC1S7+Qpo9m2 h6Q7q82Sy/re1ZpO5X5TfePmKQkKw/OPSKuMg4oLGTibpFRChVkViXayzGu1Ntp+M7Ze1++k VSvFj+aGzYo7uscl2cx2fwcb1WlZsFPnz8MCigcHFc1Vt27nSQpHY3oUuSRPGMkD6iiXCx8r KCmquPRNeHeGmg7Z7RzyPHcNIc2p6ysIedzGfst0H6CwxrmGT8SmB9Vx8JKPEFN3rG3Tl6+G bAe88s4q2NZAY0Oc0WX1AtFqoPz48KFBug+ID2qdGb8/mhSPDTvuDxhUYKCwNKqLdSpVB1Yk rYYYXaweL8Am72GVOnMo3vrgPb9Et7zzy9HZ+5OdZpQq5t06SyVHcaQr5nGUNyhVpo9Y/aPv jRhJDqfaYJNziiSdFQnatINXDqr2auWnWbCtiY9JUXOOCizh+M2QDdAeJEFchJxhxqU7XBVj k7TBTaeOJVIW8i/o5qCyYGMDaXzpQXnQ1AYIPLrhlNtdLfPyMJUUBDThqo5bwlQIN1hZTjhs m1BiDJ8fvN3BJbkSCkVoWSYYxT5I/72WtBUA1HNt2f2/W1P2NgDId8TyAVAwG28Dgen8Q2HI /Ntbn4DHqR+KJrTWYzb6QJFdJNdJukjUjOAcwMV57NlHeTZIyXxLAiDvlYeM8byTnGb0+uTo NZ27+Acgtk+nDXXYCpNjCVFUXh3suTxxiDbl3hOHeJRHHTfEI0It5sMol8jOjRSp4U7obNCQ /jTPpxCd1NEv1eOegypm/35NDnG0i6crO7ZnO+xVUx5pbDRX58XoeLmcUhklHLU6QPVDKdS/ 7+V82yxdRs94WE07rAnd9PPqNsqvP7sKffbRYx+tVgkrzzdif6vZxQ+rg1P1Y2DkkdiwSj8r kaAdQOmDGudVqpNwJWD08o1br+QbQfCjGVzo6swD0KyXIRGx8iTQGqRkCK+Oydwz7yrwyu9K TQAXKFLTzKuzRuX5Pjzeh/9cho8iUAEcR8s3utRArF+sQSf2DUMVdw3tro9rOQVU55qx+nUE MhjrePfWIsphEmhrf6O9whZ9nYx3SnTkr6PvwdSfXOho6N3b4x9H7y7OT45+coyQWj3tOXD3 dyMx8/AElOA5UgkymfINM+31LX1WrLOqic7j648wspIKW8H2VHpy3y5ju7aJla2Auu7VuLyG jW38MBqkpHScde2lzhu7Uyr50nsMzXOUJTuhIx5iwxrzN8zGYMA24iYdNDbwTvgCb9c3CWSE YRSuZOag+75kg8bRq6mvDxQqfCiU6jsgM7/++ivVU+QeFMAeozek8yV0R8Tp4n+a6TmGQAjV aYlfpCgqluPBGyJq6K50WrEKx80xamVlfk2FuHazgWtrP1eqxXBdPdWMtNTyJMGAxr9lO2Sm Kz60LtdgU7+ezMQ1aMkqtL7HNoEbNMAJvnH23kqjDkdB7BARq2X1WgxVIVgKc84/mvtGOssB mrakbirFwSAcB24M6yRXq6AbX+vVZRNw6kUi7wLrZaYPsTu2kmdAuCqNXUnu8G7dHJCEK2dS TzrW6I8O619uiurxCGtTjyiL2dUAnI00Ho0gH1BBo4FUWEdpJVpoRDm4UcU6FJdKj/gsdEo9 qAY1EbHB7z4L1XbWqp2H6FW9m22ErzArHZ16pjBMA7zbrT/QZVS3Vt4Cd1k9w6se3W08laM8 EkjFFRVe7FlgezBAFmAU0mpeFz9madipjgDSuCF9At4R0hZfhoJPKAJ5RqOZHyWjka1if7kk 8/1y50WSUNEZa3ueXTXJR0o3La3tBXefhB5aTvJ0RPuVqxWIx+4QV8xnHXXAuxpIUzgtgmJu xN7zur7HvbKPFobl1m61RqfbfCq4/gq/ja/x2/wqv5bX+a0NbVfta+1df+t3+mWm+X2aAheK wLB+K3li22MoV+SeaXemMbDVVJh8OqombY7l/9SpKR3XdVa26wxBoh7dCpjjrT/8X2I8lHDx x+aN5dWXIvojeUYX9Qgfu8cHJEJw7uqu0zIiGamju8aOWFQXSxsLFrY8htDpv3y533PWS6g6 iI5S2TXFyHiwn8A594rSA6SmMbfe0zGml10ae3smnQwKXCo7sQq2U5ZMqAbnrMAnjStfeKD6 jtRLERRQdHRaDjpO6xsthZS4Lhu3Y4LnNo25P0qtlSeda5zsa0Z+RIE62P9D7+BsZW8bBvLI 6UYMBn8IA8t6ym4iSLlyMTxgYgEfYP287bvwttf22l7ba3ttr+21vbbX9tpe22t7ba/ttb22 1/baXttre22v7bW9ttf2+je//g8J8zllAHgAAA== --4Ckj6UjgE2iN1+kY-- From lisa@gentoo.org Fri Aug 8 03:29:30 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: 07 Aug 2003 22:29:30 -0400 Subject: python api, try two In-Reply-To: <20030808022321.GC17601@danga.com> References: <20030808022321.GC17601@danga.com> Message-ID: <1060309770.1765.17.camel@lisa.thedoh.com> --=-rvyOkoDBbwMoPSqYkVcx Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Thu, 2003-08-07 at 22:23, Evan Martin wrote: > This is 1.1. > * Add incr, decr, and delete. > * Better Python (based on comments from Uriah Welcome). > * Docs, using epydoc. >=20 > Hopefully it's ready for CVS now. When this makes it to the API download page it'll be available for Gentoo users. Thanks for your hard work, Evan! --=20 Regards, -Lisa --=-rvyOkoDBbwMoPSqYkVcx Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/MwsKy0a1Vh5Jb8URAq7YAKCADwJoCQ1w3sawkIWY836OMsQ2ugCgnpa7 h9NQZH/VKAKspaSJ2qgHM5I= =AF// -----END PGP SIGNATURE----- --=-rvyOkoDBbwMoPSqYkVcx-- From brad@danga.com Fri Aug 8 08:39:38 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 00:39:38 -0700 (PDT) Subject: memcached in freebsd Message-ID: I noticed FreeBSD has a number of patches to memcached in their ports tree. I'm wondering if we shouldn't try and get those autoconfed into the main tree, to make it easier on the other BSDs? (I'm so not the person to do that, though, so this is more of a cry for help...) - Brad From brad@danga.com Fri Aug 8 19:44:28 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 11:44:28 -0700 (PDT) Subject: memcached in freebsd In-Reply-To: References: Message-ID: BTW, those files are at: http://www.freebsd.org/cgi/cvsweb.cgi/ports/databases/memcached/files/ Anybody have access to both Linux and FreeBSD and knows autoconf? I guess we can't do mallinfo on FreeBSD, and their mlockall support isn't totally done yet. (Oddly, somebody I read on LiveJournal has been working on mlockall support for FreeBSD....) On Fri, 8 Aug 2003, Brad Fitzpatrick wrote: > I noticed FreeBSD has a number of patches to memcached in their ports > tree. > > I'm wondering if we shouldn't try and get those autoconfed into the main > tree, to make it easier on the other BSDs? > > (I'm so not the person to do that, though, so this is more of a cry for > help...) > > - Brad > > > From brad@danga.com Fri Aug 8 20:06:02 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 12:06:02 -0700 (PDT) Subject: gentoo roadmap Message-ID: Thoughts on the Gentoo memcached roadmap[1]: * The APIs shouldn't RDEPEND on memcached. In LiveJournal's case, our memcached machines are on totally separate hosts from our web nodes. * I'd like the init/conf scripts included in memcached. It's not so much that I'm lazy, but I'm not sure the correct way to do it. I'd just write it in Perl, but probably wouldn't fly with most people. However it is, we need to support multiple processes in the conf file. For instance, we have a 12 GB machine where we run 5x2 GB processes, leaving 2 GB on the machine for other stuff. Because it's a 32 bit machine, we can't just run 1 10 GB process. Anybody using memcached seriously is going to want these same sorts of configs. - Brad [1] http://dev.gentoo.org/~lisa/memcached/roadmap.txt From lisa@gentoo.org Fri Aug 8 20:19:05 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: 08 Aug 2003 15:19:05 -0400 Subject: gentoo roadmap In-Reply-To: References: Message-ID: <1060370345.1765.30.camel@lisa.thedoh.com> --=-6+RYBuef5Bc2zLIunR0B Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Fri, 2003-08-08 at 15:06, Brad Fitzpatrick wrote: > Thoughts on the Gentoo memcached roadmap[1]: >=20 > * The APIs shouldn't RDEPEND on memcached. In LiveJournal's case, our > memcached machines are on totally separate hosts from our web nodes. I had debated this with myself before I put the ebuilds in. I came to the conclusion that memcached is not a large package, and it compiles very quickly. It isn't too much of a big deal; I'll remove the depends. > * I'd like the init/conf scripts included in memcached. It's not so much > that I'm lazy, but I'm not sure the correct way to do it. I'd just > write it in Perl, but probably wouldn't fly with most people. The init scripts are in the image of Bash, so we can't use Perl. > However it is, we need to support multiple processes in the conf file. > For instance, we have a 12 GB machine where we run 5x2 GB processes, > leaving 2 GB on the machine for other stuff. Because it's a 32 bit > machine, we can't just run 1 10 GB process. This seems like a candidate for program enhancement: memcached -d --servers 5 --mem 2048 =3D 5 servers at 2048MB each. Mainly because I don't particularly relish the job of modifying the init script. ;-) I'll see what I can do... --=20 Regards, -Lisa --=-6+RYBuef5Bc2zLIunR0B Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/M/epy0a1Vh5Jb8URAr5MAKCuMOHw/JV2KY0BrUe36PVkhw+TZgCeNT4y 9SsEkOMbuLtK/uipgO8Rgpo= =QKCP -----END PGP SIGNATURE----- --=-6+RYBuef5Bc2zLIunR0B-- From brad@danga.com Fri Aug 8 20:38:14 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 12:38:14 -0700 (PDT) Subject: gentoo roadmap In-Reply-To: <1060370345.1765.30.camel@lisa.thedoh.com> References: <1060370345.1765.30.camel@lisa.thedoh.com> Message-ID: Also, how do you handle libevent? Do you build libevent with epoll, even if the kernel doesn't support it? Because libevent selects poll/select/epoll/kqueue at runtime, it's best to build the most powerful libevent as possible, so users can upgrade their kernel later. In my experience, building the kernel with epoll is way easier than fixing all the userspace headers. How does Gentoo handle all that? Is epoll* syscalls in the glibc headers you guys distribute? (note to others: memcached runs fine with poll/select. you don't need to rebuild your kernel. but it's pretty cool if you do.) On Fri, 8 Aug 2003, Lisa Marie Seelye wrote: > On Fri, 2003-08-08 at 15:06, Brad Fitzpatrick wrote: > > Thoughts on the Gentoo memcached roadmap[1]: > > > > * The APIs shouldn't RDEPEND on memcached. In LiveJournal's case, our > > memcached machines are on totally separate hosts from our web nodes. > > I had debated this with myself before I put the ebuilds in. I came to > the conclusion that memcached is not a large package, and it compiles > very quickly. > > It isn't too much of a big deal; I'll remove the depends. > > > * I'd like the init/conf scripts included in memcached. It's not so much > > that I'm lazy, but I'm not sure the correct way to do it. I'd just > > write it in Perl, but probably wouldn't fly with most people. > > The init scripts are in the image of Bash, so we can't use Perl. > > > > However it is, we need to support multiple processes in the conf file. > > For instance, we have a 12 GB machine where we run 5x2 GB processes, > > leaving 2 GB on the machine for other stuff. Because it's a 32 bit > > machine, we can't just run 1 10 GB process. > > This seems like a candidate for program enhancement: > > memcached -d --servers 5 --mem 2048 = 5 servers at 2048MB each. > > Mainly because I don't particularly relish the job of modifying the init > script. ;-) I'll see what I can do... > > > -- > Regards, > -Lisa > > From brad@danga.com Fri Aug 8 20:42:37 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 12:42:37 -0700 (PDT) Subject: gentoo roadmap In-Reply-To: <1060370345.1765.30.camel@lisa.thedoh.com> References: <1060370345.1765.30.camel@lisa.thedoh.com> Message-ID: On Fri, 8 Aug 2003, Lisa Marie Seelye wrote: > > However it is, we need to support multiple processes in the conf file. > > For instance, we have a 12 GB machine where we run 5x2 GB processes, > > leaving 2 GB on the machine for other stuff. Because it's a 32 bit > > machine, we can't just run 1 10 GB process. > > This seems like a candidate for program enhancement: > > memcached -d --servers 5 --mem 2048 = 5 servers at 2048MB each. > > Mainly because I don't particularly relish the job of modifying the init > script. ;-) I'll see what I can do... The problem is, when you have a huge memcached farm, and you want to upgrade to a new version of memcached, you don't want to stop/start everything at once... you want to do one process at a time. Having them all parent/child off each other makes it really hard. Plus, our goal is to write in C as little as possible. No config parsing, etc. My goal was always to write a Perl wrapper to control them. Couldn't we just make memcached depend on Perl? Then the init scripts could just do stop-start-daemons on the wrapper? - Brad From lisa@gentoo.org Fri Aug 8 20:57:03 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: 08 Aug 2003 15:57:03 -0400 Subject: gentoo roadmap In-Reply-To: References: <1060370345.1765.30.camel@lisa.thedoh.com> Message-ID: <1060372622.1762.42.camel@lisa.thedoh.com> --=-tikSy7YwScZQeS7bgW4U Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Fri, 2003-08-08 at 15:42, Brad Fitzpatrick wrote: > The problem is, when you have a huge memcached farm, and you want to > upgrade to a new version of memcached, you don't want to stop/start > everything at once... you want to do one process at a time. Having them > all parent/child off each other makes it really hard. >=20 > Plus, our goal is to write in C as little as possible. No config > parsing, etc. My goal was always to write a Perl wrapper to control them= . >=20 > Couldn't we just make memcached depend on Perl? Then the init scripts > could just do stop-start-daemons on the wrapper? That sounds like the best way to go -- a Perl wrapper to launch and terminate multiple instances of the memcache daemon. (Even with each instance having separate configurations... handy!) A Perl wrapper would still be called with start-stop-daemon, but would allow for a lot more control and flexibility. I'll respond to your other email here, too, so as to not keep two threads going. To the best of my knowledge the epoll kernel patch is not included with any Gentoo kernels (Not gentoo-sources, for certain). I've submitted a request to one of our Kernel guys (Bug #25671 @ bugs.gentoo.org) but I don't know when this will be handled. So libevent is built as it would normally be when /dev/epoll was not present. I'd assume that since epoll isn't included with the kernel that glibc isn't made "aware" of it (headers). After things settle down from LWE I'll ask around a bit. --=20 Regards, -Lisa --=-tikSy7YwScZQeS7bgW4U Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/NACOy0a1Vh5Jb8URAp2DAJ4jfeGTivvDHCqJxvG4i6CSvkf/LgCfQcAg z476bhuiAzFnmRfY+kgUX84= =sZ/q -----END PGP SIGNATURE----- --=-tikSy7YwScZQeS7bgW4U-- From brad@danga.com Fri Aug 8 22:12:24 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 14:12:24 -0700 (PDT) Subject: gentoo roadmap In-Reply-To: <1060372622.1762.42.camel@lisa.thedoh.com> References: <1060370345.1765.30.camel@lisa.thedoh.com> <1060372622.1762.42.camel@lisa.thedoh.com> Message-ID: > That sounds like the best way to go -- a Perl wrapper to launch and > terminate multiple instances of the memcache daemon. (Even with each > instance having separate configurations... handy!) > > A Perl wrapper would still be called with start-stop-daemon, but would > allow for a lot more control and flexibility. I'll work on that, then. > I'll respond to your other email here, too, so as to not keep two > threads going. > > To the best of my knowledge the epoll kernel patch is not included with > any Gentoo kernels (Not gentoo-sources, for certain). I've submitted a > request to one of our Kernel guys (Bug #25671 @ bugs.gentoo.org) but I > don't know when this will be handled. /dev/epoll is the 'old' way. The new way that Linus approved (which went in 2.5) is the system call. I'm not sure if libevent supports /dev/epoll. On their website they say support for /dev/poll is planned, which is the Solaris equivalent of /dev/epoll? > So libevent is built as it would normally be when /dev/epoll was not > present. I'd assume that since epoll isn't included with the kernel > that glibc isn't made "aware" of it (headers). After things settle down > from LWE I'll ask around a bit. In any case, the headers should be updated in Gentoo for the new system call numbers, so programs like libevent can build with all their functionality, even if the current running kernel doesn't support it. BTW, to test if memcached (rather, libevent) is using epoll: # EVENT_SHOW_METHOD=1 memcached From brad@danga.com Fri Aug 8 23:38:57 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 15:38:57 -0700 (PDT) Subject: Python API release Message-ID: Python supports joins Perl and PHP, thanks to Evan: http://www.danga.com/memcached/apis.bml SourceForge people: let us know if you have any questions/problems with it. - Brad From brad@danga.com Fri Aug 8 23:44:12 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 15:44:12 -0700 (PDT) Subject: Everything2 and memcached Message-ID: Jay gave me permission to repost this conversation here. Looks like the Everything2.com people will be adding memcache support, because all their web processes are full of cached data that isn't shared. (the first limit people hit... then you do IPC::Shareable or something, and hit per-machine limits...) Anyway, good to see more people using it. I haven't heard from most people on this list about why they're using it, though. Current users seem to include: LiveJournal (main user) Slashdot (using it for 15 GB+ of comments, adding more) SourceForge (starting to work on it? (python support)) Everything (starting to work on it) I'm not sure why it's in FreeBSD or Gentoo. Who's using it there? - Brad ---------- Forwarded message ---------- Date: 08 Aug 2003 14:35:37 -0400 From: Jay Bonci To: Brad Fitzpatrick Subject: Re: memcached and custom kernels Right now we cache pre-made objects into webserver proc memory (use vars $foo sorta stuff). It's not pretty and makes it memory intensive for us. Memcached is a way for us to cut down our memory footprint, and distribute some of the object hell going forward. I'll let you know when we have that all subclassed in and stable =) Thanks --jay On Fri, 2003-08-08 at 14:30, Brad Fitzpatrick wrote: > Let me know when you do and I'll add you guys to our as-of-yet-unmade > "Users Page". > > If you're having any database problems now, I can't recommend memcached > enough. It was a night & day difference for us. We hated blocking so > much, that memcached itself never blocks for anything. > > - Brad > > > On Fri, 8 Aug 2003, Jay Bonci wrote: > > > Thanks for getting back to me. Looks like we'll be putting in memcached > > support for ecore's nodecache, as it makes a lot of sense (when I get > > the free time). > > > > Slashdot's using it now, so hey, good enough for them... ;) > > > > Thanks again, > > > > > > --jay > > > > On Fri, 2003-08-08 at 03:58, Brad Fitzpatrick wrote: > > > Accidentally ran across an old mailing list post of yours. > > > > > > Just mailing you to let you know you don't need a custom whiz-bang kernel > > > to run memcached. Memcache only depends on libevent, which selects > > > between select, poll, kqueue, and epoll at run time. Linux 2.5 supports > > > epoll. There are patches to make 2.4 do it, which we use. > > > > > > There's also a patch to libevent (going in after 0.7a?) that adds realtime > > > signal support, wihch 2.4 supports, and is almost as fast as epoll. > > > > > > But the point is: memcached itself will run with anything. select/poll > > > suck a little, but they're perfectly acceptable. But if you want to > > > use hardly any CPU, definitely go the epoll route. (or kqueue on FreeBSD) > > > > > > - Brad > > From brad@danga.com Sat Aug 9 00:52:08 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 16:52:08 -0700 (PDT) Subject: website updates Message-ID: I added a note about memcached vs. shared memory, after recently reading somebody on a mailing list debating between the two: http://www.danga.com/memcached/ And also a users page: http://www.danga.com/memcached/users.bml If you're on the users page and hate your description or want to be removed, let me know. Also, if you're not on there and use memcached, let me know. I want to link the CVS browse page for the CONTRIBUTORS file, but I can't decide where to link it from. Suggestions? - Brad From martine@danga.com Sat Aug 9 01:48:23 2003 From: martine@danga.com (Evan Martin) Date: 08 Aug 2003 17:48:23 -0700 Subject: gentoo roadmap In-Reply-To: References: <1060370345.1765.30.camel@lisa.thedoh.com> Message-ID: <1060390103.14780.34.camel@trout> On Fri, 2003-08-08 at 12:42, Brad Fitzpatrick wrote: > Plus, our goal is to write in C as little as possible. No config > parsing, etc. My goal was always to write a Perl wrapper to control them. > > Couldn't we just make memcached depend on Perl? Then the init scripts > could just do stop-start-daemons on the wrapper? It would seem natural to write them in as shell scripts. Then, there's no dependencies at all. Or am I missing something? -- Evan Martin martine@danga.com http://neugierig.org From brad@danga.com Sat Aug 9 01:55:47 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Fri, 8 Aug 2003 17:55:47 -0700 (PDT) Subject: gentoo roadmap In-Reply-To: <1060390103.14780.34.camel@trout> References: <1060370345.1765.30.camel@lisa.thedoh.com> <1060390103.14780.34.camel@trout> Message-ID: I don't do bash-foo. I prefer the pain of perl. It's accustomed pain. On Fri, 8 Aug 2003, Evan Martin wrote: > On Fri, 2003-08-08 at 12:42, Brad Fitzpatrick wrote: > > Plus, our goal is to write in C as little as possible. No config > > parsing, etc. My goal was always to write a Perl wrapper to control them. > > > > Couldn't we just make memcached depend on Perl? Then the init scripts > > could just do stop-start-daemons on the wrapper? > > It would seem natural to write them in as shell scripts. Then, there's > no dependencies at all. > > Or am I missing something? > > -- > Evan Martin > martine@danga.com > http://neugierig.org > > From ryan@gilfether.com Sat Aug 9 18:08:13 2003 From: ryan@gilfether.com (Ryan Gilfether) Date: Sat, 09 Aug 2003 13:08:13 -0400 Subject: PHP API 1.0.7 Message-ID: <3F352A7D.9090306@gilfether.com> A new version of the PHP API has been completed, verson 1.0.7. I've had some requests for adding PHPDocs style comments to the API. Along with that 3 new function have been added error() - returns an error code for the last error genterated error_string() - returns a string with a description of the error error_clear() - clears the last error Ryan -- Ryan Gilfether Software Engineer http://www.gilfether.com From brion@pobox.com Sun Aug 10 07:49:47 2003 From: brion@pobox.com (Brion Vibber) Date: Sat, 09 Aug 2003 23:49:47 -0700 Subject: PHP API 1.0.7 In-Reply-To: <3F352A7D.9090306@gilfether.com> References: <3F352A7D.9090306@gilfether.com> Message-ID: <3F35EB0B.7010506@pobox.com> This is a multi-part message in MIME format. --------------060402060905030606060808 Content-Type: text/plain; charset=us-ascii; format=flowed Content-Transfer-Encoding: 7bit Ryan Gilfether wrote: > A new version of the PHP API has been completed, verson 1.0.7. Cool! Where can I find it? I'm doing some preliminary work on getting memcached support into Wikipedia (or rather MediaWiki, as we've recently redubbed the software thereof), so far I've started with the 1.0.6 PHP client. > Along with that 3 new function have been added > error() - returns an error code for the last error genterated > error_string() - returns a string with a description of the error > error_clear() - clears the last error On a related note, the debug mode doesn't mix well with MediaWiki's output model. I replaced all the "print"s with calls to a _debug() private function which can I can then override in a subclass to dump things to our debug log; diff for 1.0.6 attached. -- brion vibber (brion @ pobox.com) --------------060402060905030606060808 Content-Type: text/plain; name="php-memc-debug.diff" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="php-memc-debug.diff" --- /home/brion/src/php-memcached-1.0.6/MemCachedClient.inc.php Mon Jul 28 19:04:24 2003 +++ MemCachedClient.inc.php Sat Aug 9 23:35:19 2003 @@ -74,7 +74,7 @@ if(!$this->active) { if($this->debug) - print "delete(): There are no active servers available\r\n"; + $this->_debug("delete(): There are no active servers available"); return FALSE; } @@ -104,7 +104,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_delete(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_delete(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -117,7 +117,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_delete(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_delete(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -131,7 +131,7 @@ return TRUE; if($this->debug) - print "_delete(): Failed to receive DELETED response from server. Received $retval instead.\r\n"; + $this->_debug("_delete(): Failed to receive DELETED response from server. Received $retval instead."); return FALSE; } @@ -220,7 +220,7 @@ if($this->debug) { while(list($k, $v) = each($val)) - print "MemCache: got $k = $v\n"; + $this->_debug("MemCache: got $k = $v"); } return $val; @@ -263,7 +263,7 @@ if(count($conn) != 2) { if($this->debug) - print "sock_to_host(): Host address was not in the format of host:port\r\n"; + $this->_debug("sock_to_host(): Host address was not in the format of host:port"); return FALSE; } @@ -272,7 +272,7 @@ ($this->host_dead[$conn[0]] && $this->host_dead[$conn[0]] > $now)) { if($this->debug) - print "sock_to_host(): The host $host is not available.\r\n"; + $this->_debug("sock_to_host(): The host $host is not available."); return FALSE; } @@ -286,7 +286,7 @@ $this->host_dead[$host]=$this->host_dead[$conn[0]]=$now+60+intval(rand(0, 10)); if($this->debug) - print "sock_to_host(): Failed to connect to ".$conn[0].":".$conn[1]."\r\n"; + $this->_debug("sock_to_host(): Failed to connect to ".$conn[0].":".$conn[1]); return FALSE; } @@ -304,7 +304,7 @@ if(!$this->active) { if($this->debug) - print "get_sock(): There are no active servers available\r\n"; + $this->_debug("get_sock(): There are no active servers available"); return FALSE; } @@ -344,7 +344,7 @@ } if($this->debug) - print "get_sock(): get_sock(): Unable to retrieve a valid socket\r\n"; + $this->_debug("get_sock(): get_sock(): Unable to retrieve a valid socket"); return FALSE; } @@ -359,7 +359,7 @@ if(!$this->active) { if($this->debug) - print "_incrdecr(): There are no active servers available\r\n"; + $this->_debug("_incrdecr(): There are no active servers available"); return FALSE; } @@ -368,7 +368,7 @@ if(!is_resource($sock)) { if($this->debug) - print "_incrdecr(): Invalid socket returned by get_sock()\r\n"; + $this->_debug("_incrdecr(): Invalid socket returned by get_sock()"); return FALSE; } @@ -394,7 +394,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_incrdecr(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_incrdecr(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -405,7 +405,7 @@ if(($retval = socket_read($sock, MC_BUFFER_SZ, PHP_NORMAL_READ)) === FALSE) { $errno = socket_last_error($sock); - print "_incrdecr(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_incrdecr(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)); } // strip the /r/n from the end @@ -424,7 +424,7 @@ if(!$this->active) { if($this->debug) - print "_set(): There are no active servers available\r\n"; + $this->_debug("_set(): There are no active servers available"); return FALSE; } @@ -433,7 +433,7 @@ if(!is_resource($sock)) { if($this->debug) - print "_set(): Invalid socket returned by get_sock()\r\n"; + $this->_debug("_set(): Invalid socket returned by get_sock()"); return FALSE; } @@ -472,7 +472,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_set(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_set(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -485,7 +485,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_set(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_set(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -494,13 +494,13 @@ if($l_szResponse == "STORED") { if($this->debug) - print "MemCache: $cmdname $key = $raw_val\n"; + $this->_debug("MemCache: $cmdname $key = $raw_val"); return TRUE; } if($this->debug) - print "_set(): Did not receive STORED as the server response! Received $l_szResponse instead\r\n"; + $this->_debug("_set(): Did not receive STORED as the server response! Received $l_szResponse instead"); return FALSE; } @@ -536,7 +536,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_load_items(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_load_items(): socket_write() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -556,7 +556,7 @@ if($this->debug) { $errno = socket_last_error($sock); - print "_load_items(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)."\r\n"; + $this->_debug("_load_items(): socket_read() returned FALSE. Error $errno: ".socket_strerror($errno)); } return FALSE; @@ -584,7 +584,7 @@ { // something went wrong, we never recieved the header if($this->debug) - print "_load_items(): Failed to recieve valid header!\r\n"; + $this->_debug("_load_items(): Failed to recieve valid header!"); return FALSE; } @@ -625,7 +625,7 @@ else { if($this->debug) - print "_load_items(): Failed to receive END. Received $end instead.\r\n"; + $this->_debug("_load_items(): Failed to receive END. Received $end instead."); return FALSE; } @@ -635,7 +635,7 @@ if($bytes_read > ($len + 7)) { if($this->debug) - print "_load_items(): Bytes read is greater than requested data size!\r\n"; + $this->_debug("_load_items(): Bytes read is greater than requested data size!"); return FALSE; } @@ -656,6 +656,10 @@ return $hash; } + + function _debug($text) { + print $text . "\r\n"; + } } /* @@ -672,12 +676,3 @@ * http://www.gilfether.com */ ?> - - - - - - - - - --------------060402060905030606060808-- From brion@pobox.com Sun Aug 10 08:17:24 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 00:17:24 -0700 Subject: memcached in freebsd Message-ID: <3F35F184.7050205@pobox.com> *Brad Fitzpatrick wrote:* >BTW, those files are at: > >http://www.freebsd.org/cgi/cvsweb.cgi/ports/databases/memcached/files/ > >Anybody have access to both Linux and FreeBSD and knows autoconf? > > Well, I happen to have both Linux and FreeBSD boxen that I'm testing on, but I'm no autoconf guru. On my FreeBSD 5.1 box I managed to install the above-linked port of memcached 1.1.6. It compiles, installs, and runs, and seems to be willing to _store_ items, but when I try to _retrieve_ something it fails: _load_items(): Failed to recieve valid header! I haven't gotten 1.1.8 to compile on FreeBSD yet. 1.1.8 running on my Linux box (Red Hat 8.0) both stores and retrieves, which is good. :) I'm using the 1.0.6 PHP client on the FreeBSD box, with PHP 4.3.2. Is there any way to get memcached to print out some info on what it's doing? I thought I'd put it into verbose mode ("memcached -v -p 11000") but it's eerily silent, printing neither error nor success messages. -- brion vibber (brion @ pobox.com) From martine@danga.com Sun Aug 10 09:44:45 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 01:44:45 -0700 Subject: memcached in freebsd In-Reply-To: <3F35F184.7050205@pobox.com> Message-ID: On Sunday, August 10, 2003, at 12:17 AM, Brion Vibber wrote: > On my FreeBSD 5.1 box I managed to install the above-linked port of > memcached 1.1.6. It compiles, installs, and runs, and seems to be > willing to _store_ items, but when I try to _retrieve_ something it > fails: > > _load_items(): Failed to recieve valid header! The memcache protocol works over telnet, which is nice for debugging. Try something like: % telnet my.host 11211 after connecting, type: set test 0 0 5 hello and you should get STORED back. then try get test and see if you get "hello" back. From brion@pobox.com Sun Aug 10 10:42:38 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 02:42:38 -0700 Subject: memcached in freebsd In-Reply-To: References: Message-ID: <3F36138E.50503@pobox.com> Evan Martin wrote: > On Sunday, August 10, 2003, at 12:17 AM, Brion Vibber wrote: > >> On my FreeBSD 5.1 box I managed to install the above-linked port of >> memcached 1.1.6. It compiles, installs, and runs, and seems to be >> willing to _store_ items, but when I try to _retrieve_ something it >> fails: >> >> _load_items(): Failed to recieve valid header! > > > The memcache protocol works over telnet, which is nice for debugging. > Try something like: On further investigation, I note two things: a) The "_load_items(): Failed to recieve valid header!" is what I get anytime I ask for data that isn't available, and seems to be normal. b) It _does_ retrieve data back from the BSD box if I don't set an expiration time when doing sets/adds. I had been using an expiration time of 3600, which should be one hour if I'm reading the docs right. ... D'oh! Looks like relative expiration times are a new feature of 1.1.7/1.1.8, and 1.1.6 expects either 0 or a full Unix timestamp. Okay, if I set it to 3600 + time(), then it gives me my data back from both boxen. Hoo-rah... at least it seems to be working on FreeBSD, it's just a version behind. :) -- brion vibber (brion @ pobox.com) From mellon@pobox.com Sun Aug 10 14:11:20 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Sun, 10 Aug 2003 13:11:20 +0000 Subject: memcached in freebsd In-Reply-To: <3F36138E.50503@pobox.com> References: <3F36138E.50503@pobox.com> Message-ID: <20030810131120.GA30897@pobox.com> Brion Vibber wrote on Sun, Aug 10, 2003 at 02:42:38AM -0700: > On further investigation, I note two things: > > a) The "_load_items(): Failed to recieve valid header!" is what I get > anytime I ask for data that isn't available, and seems to be normal. Well, if some data you asked for isn't available, it's _not_ an error, and you don't fail to receive a valid header, so something strange is going on. A cursory look at the client's source (I don't really know any PHP) seems to indicate that the client makes some unwarranted assumptions. I _think_ it only recognises the "END\r\n" line which finishes any batch of data a server sends when this line comes in the same buffer (as a result of the same read call) with some previous data for some key. As a result, when the server fails to match any keys, and only sends "END\r\n" (correctly, according to the protocol), the client thinks it's an error. This by itself may be harmless, but I think that more serious errors could occur if the "END\r\n" doesn't come in the same IP packet as the rest of server's reply (say if the reply is too large to fit in one packet) and subsequently arrives via a separate read request. This logic should be fixed (there shouldn't be any assumptions about how data is separated across read/write calls). > D'oh! Looks like relative expiration times are a new feature of > 1.1.7/1.1.8, and 1.1.6 expects either 0 or a full Unix timestamp. That's correct (don't use 1.1.7, it has an embarrassing bug related to this very feature). --avva From brion@pobox.com Sun Aug 10 11:32:23 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 03:32:23 -0700 Subject: Fast writes, slow reads Message-ID: <3F361F37.8010204@pobox.com> This is a multi-part message in MIME format. --------------040400050004020103050504 Content-Type: text/plain; charset=us-ascii; format=flowed Content-Transfer-Encoding: 7bit I'm finding that reading data back from memcached is surprisingly slow. I've probably done something wrong. :) I wrote a simple loop that reads 500 integers or strings, adding each one to the cache if it's not already in. The first run, which checks for 500 uncached items and adds 500 items to the cache, runs fast enough; more or less a second. The second run, which fetches back the set values, consistently takes about 50 seconds, about a tenth of a second per fetched item. 1000 items take 100 seconds, 50 items take 5 seconds. Since I expect to do a lot more reading than writing, this has me a little worried! I'm getting this same slow read speed from the PHP and Perl clients, and both talking to a local server on the BSD box or over the network to the Linux box. Moving the Perl client from the 2 GHz Athlon (BSD) to the 266 MHz Pentium II (Linux), oddly enough my speeds increase a bit: it takes a mere 20 seconds to fetch 500 items (though the check + add sequence goes up to 2 seconds). It doesn't seem to make a difference whether it's talking to the local or the remote server. Any ideas? I've attached the Perl version of the test script; I'm using the 1.0.6 MemCachedClient... Perl 5.6.1 is installed on the FreeBSD box, 5.8.0 on the Linux box. -- brion vibber (brion @ pobox.com) --------------040400050004020103050504 Content-Type: text/plain; name="mctest.pl" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="mctest.pl" #!/usr/bin/perl # memcached -p 11000 use MemCachedClient; $wgMemc = new MemCachedClient { 'servers' => [ "127.0.0.1:11000" ], # 'servers' => [ "10.0.0.1:11000" ], # 'servers' => [ "10.0.0.2:11000" ], 'debug' => 0 }; $max = 500; print "Grab $max integers:\n"; $time_start = time(); @arr = []; for($i=0; $i<$max; $i++) { $n = $wgMemc->get( $key = "test:intarray:$i" ); if(!$n) { print "* $i not in cache, adding\n"; $n = $i; $wgMemc->add( $key, $n ); } else { print "* $n found\n"; } $arr[$i] = $n; } $time_end = time(); $time = $time_end - $time_start; print "Retrieved $max ints in $time secs.\n\n"; print "Grab $max strings:\n"; $time_start = time(); @arr = []; for($i=0; $i<$max; $i++) { $n = $wgMemc->get( $key = "test:stringarray:$i" ); if(!$n) { print "* $i not in cache, adding\n"; $n = "this is the ${i}th string"; $wgMemc->add( $key, $n ); } else { print "* $n found\n"; } $arr[$i] = $n; } $time_end = time(); $time = $time_end - $time_start; print "Retrieved $max strings in $time secs.\n\n"; --------------040400050004020103050504-- From martine@danga.com Sun Aug 10 11:40:33 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 03:40:33 -0700 Subject: [patch] autoconf cleanups, build on osx (incl. freebsd patches) Message-ID: <12249ABF-CB1F-11D7-B0B6-003065B8A5CC@danga.com> --Apple-Mail-2-600455367 Content-Disposition: attachment; filename=memcached-bsd.patch Content-Transfer-Encoding: 7bit Content-Type: application/octet-stream; x-unix-mode=0644; name="memcached-bsd.patch" Index: ChangeLog =================================================================== RCS file: /home/cvspub/wcmtools/memcached/ChangeLog,v retrieving revision 1.11 diff -u -r1.11 ChangeLog --- ChangeLog 30 Jul 2003 05:53:49 -0000 1.11 +++ ChangeLog 10 Aug 2003 10:38:20 -0000 @@ -1,3 +1,14 @@ +2003-08-10 (Evan Martin) + * Makefile.am: debug, optimization, and static flags are controlled + by the configure script. + * configure.ac: + - allow specifying libevent directory with --with-libevent=DIR + - check for malloc.h (unavailable on BSDs) + - check for socklen_t (unavailable on OSX) + * assoc.c, items.c, slabs.c: Remove some unused headers. + * memcached.c: allow for nonexistence of malloc.h; #define a POSIX + macro to import mlockall flags. + 2003-07-29 * version 1.1.7 * big bug fix: item exptime 0 meant expire immediately, not never Index: Makefile.am =================================================================== RCS file: /home/cvspub/wcmtools/memcached/Makefile.am,v retrieving revision 1.8 diff -u -r1.8 Makefile.am --- Makefile.am 7 Aug 2003 17:31:17 -0000 1.8 +++ Makefile.am 10 Aug 2003 10:38:20 -0000 @@ -5,6 +5,5 @@ DIST_SUBDIRS = doc EXTRA_DIST = doc TODO -CFLAGS=-g -O2 -static -DNDEBUG - +AM_CFLAGS=-DNDEBUG Index: assoc.c =================================================================== RCS file: /home/cvspub/wcmtools/memcached/assoc.c,v retrieving revision 1.5 diff -u -r1.5 assoc.c --- assoc.c 27 Jun 2003 21:15:55 -0000 1.5 +++ assoc.c 10 Aug 2003 10:38:20 -0000 @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -28,7 +27,6 @@ #include #include #include -#include #include #include "memcached.h" Index: configure.ac =================================================================== RCS file: /home/cvspub/wcmtools/memcached/configure.ac,v retrieving revision 1.15 diff -u -r1.15 configure.ac --- configure.ac 30 Jul 2003 05:53:49 -0000 1.15 +++ configure.ac 10 Aug 2003 10:38:20 -0000 @@ -7,6 +7,42 @@ AC_PROG_CC AC_PROG_INSTALL +AC_ARG_WITH(libevent, + AC_HELP_STRING([--with-libevent=DIRECTORY],[base directory for libevent])) +if test ${with_libevent+set} = set && test $with_libevent != no; then + CFLAGS="$CFLAGS -I$with_libevent/include" + LDFLAGS="$LDFLAGS -L$with_libevent/lib" +fi + +LIBEVENT_URL=http://www.monkey.org/~provos/libevent/ +AC_CHECK_LIB(event, event_set, , + [AC_MSG_ERROR(libevent is required. You can get it from $LIBEVENT_URL)]) + +AC_CHECK_HEADER(malloc.h, AC_DEFINE(HAVE_MALLOC_H,,[do we have malloc.h?])) + +dnl From licq: Copyright (c) 2000 Dirk Mueller +dnl Check if the type socklen_t is defined anywhere +AC_DEFUN(AC_C_SOCKLEN_T, +[AC_CACHE_CHECK(for socklen_t, ac_cv_c_socklen_t, +[ + AC_TRY_COMPILE([ + #include + #include + ],[ + socklen_t foo; + ],[ + ac_cv_c_socklen_t=yes + ],[ + ac_cv_c_socklen_t=no + ]) +]) +if test $ac_cv_c_socklen_t = no; then + AC_DEFINE(socklen_t, int, [define to int if socklen_t not available]) +fi +]) + +AC_C_SOCKLEN_T + dnl Default to building a static executable. AC_ARG_ENABLE(static, AC_HELP_STRING([--disable-static],[build a dynamically linked executable]), @@ -19,10 +55,6 @@ CFLAGS="$CFLAGS -static" AC_MSG_RESULT(yes) fi - -LIBEVENT_URL=http://www.monkey.org/~provos/libevent/ -AC_CHECK_LIB(event, event_set, , - [AC_MSG_ERROR(libevent is required. You can get it from $LIBEVENT_URL)]) AC_CONFIG_FILES(Makefile) AC_OUTPUT Index: items.c =================================================================== RCS file: /home/cvspub/wcmtools/memcached/items.c,v retrieving revision 1.18 diff -u -r1.18 items.c --- items.c 23 Jul 2003 00:00:20 -0000 1.18 +++ items.c 10 Aug 2003 10:38:20 -0000 @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -17,7 +16,6 @@ #include #include #include -#include #include #include "memcached.h" Index: memcached.c =================================================================== RCS file: /home/cvspub/wcmtools/memcached/memcached.c,v retrieving revision 1.29 diff -u -r1.29 memcached.c --- memcached.c 30 Jul 2003 05:53:49 -0000 1.29 +++ memcached.c 10 Aug 2003 10:38:22 -0000 @@ -23,6 +23,11 @@ #include #include #include +/* some POSIX systems need the following definition + * to get mlockall flags out of sys/mman.h. */ +#ifndef _P1003_1B_VISIBLE +#define _P1003_1B_VISIBLE +#endif #include #include #include @@ -34,9 +39,12 @@ #include #include #include -#include #include +#ifdef HAVE_MALLOC_H +#include +#endif + #include "memcached.h" struct stats stats; @@ -314,6 +322,7 @@ return; } +#ifdef HAVE_MALLOC_H if (strcmp(command, "stats malloc") == 0) { char temp[512]; struct mallinfo info; @@ -333,6 +342,7 @@ out_string(c, temp); return; } +#endif /* HAVE_MALLOC_H */ if (strcmp(command, "stats maps") == 0) { char *wbuf; Index: slabs.c =================================================================== RCS file: /home/cvspub/wcmtools/memcached/slabs.c,v retrieving revision 1.13 diff -u -r1.13 slabs.c --- slabs.c 27 Jul 2003 05:04:37 -0000 1.13 +++ slabs.c 10 Aug 2003 10:38:22 -0000 @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -20,7 +19,6 @@ #include #include #include -#include #include #include "memcached.h" --Apple-Mail-2-600455367 Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset=US-ASCII; format=flowed The ChangeLog: * Makefile.am: debug, optimization, and static flags are controlled by the configure script. * configure.ac: - allow specifying libevent directory with --with-libevent=DIR - check for malloc.h (unavailable on BSDs) - check for socklen_t (unavailable on OSX) * assoc.c, items.c, slabs.c: Remove some unused headers. * memcached.c: allow for nonexistence of malloc.h; #define a POSIX macro to import mlockall flags. My notes: This now builds on OSX, though I get some weird linker errors unless I --disable-static. But in theory, working on OSX means it probably works on FreeBSD. That socklen_t check looks hairy, but even libevent does a similar check. (Ours is a bit more complicated to allow autoconf caching but it's a cut'n'paste job so it should work.) Special thanks to Sean Chittenden (have we met? I swear I know that name); I used his FreeBSD patches as a reference for what doesn't work on BSD. Rather than removing blocks of code as his patches did, this patch #ifdefs out code that is platform-specific (only the slab stats stuff, really). Hopefully, mlockall actually does work on FreeBSD and the problem there was only that MCL_* weren't getting set; it looks like mlockall is in POSIX[1] and should work everywhere. The +#define _P1003_1B_VISIBLE needed to get those flags at first glance looks like a huge hack but some googling indicates it's moderately standard[2]. I'm afraid it's a derived setting from some other setting (possibly simple P1003_B, as some things I'm finding on Google indicate), but at least on OSX it's never set and only read in one file, as far as I can tell: lulu:~% grep -lrs _P1003_1B_VISIBLE /usr/include/* /usr/include/sys/mman.h In retrospect the patch is tiny, but damn did that all take me a while to figure out. Untested on Linux or FreeBSD, so comments are welcome. :) [1] http://www.opengroup.org/onlinepubs/007908799/xsh/mlockall.html [2] http://www.chedong.com/phpMan.php/man/posix4/9 -- Evan Martin martine@danga.com http://neugierig.org --Apple-Mail-2-600455367-- From martine@danga.com Sun Aug 10 11:45:24 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 03:45:24 -0700 Subject: Fast writes, slow reads In-Reply-To: <3F361F37.8010204@pobox.com> References: <3F361F37.8010204@pobox.com> Message-ID: <20030810104524.GA7244@danga.com> On Sun, Aug 10, 2003 at 03:32:23AM -0700, Brion Vibber wrote: > The second run, which fetches back the set values, consistently takes > about 50 seconds, about a tenth of a second per fetched item. 1000 items > take 100 seconds, 50 items take 5 seconds. This sounds like a problem I was having related to the backend method libevent was using. Which one are you using? To find out: % EVENT_SHOW_METHOD=1 ./memcached libevent using: select -- Evan Martin martine@danga.com http://neugierig.org From brion@pobox.com Sun Aug 10 11:52:23 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 03:52:23 -0700 Subject: Fast writes, slow reads In-Reply-To: <20030810104524.GA7244@danga.com> References: <3F361F37.8010204@pobox.com> <20030810104524.GA7244@danga.com> Message-ID: <3F3623E7.5040105@pobox.com> Evan Martin wrote: >This sounds like a problem I was having related to the backend method >libevent was using. > Uh-oh... :) > Which one are you using? To find out: >% EVENT_SHOW_METHOD=1 ./memcached >libevent using: select > FreeBSD 5.1, libevent 0.7a installed from ports: $ EVENT_SHOW_METHOD=1 memcached libevent using: kqueue Red Hat 8.0, libevent 0.7a installed from source with no special options: $ EVENT_SHOW_METHOD=1 ./memcached libevent using: poll -- brion vibber (brion @ pobox.com) From mellon@pobox.com Sun Aug 10 15:38:10 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Sun, 10 Aug 2003 14:38:10 +0000 Subject: Fast writes, slow reads In-Reply-To: <3F361F37.8010204@pobox.com> References: <3F361F37.8010204@pobox.com> Message-ID: <20030810143810.GA31414@pobox.com> You wrote on Sun, Aug 10, 2003 at 03:32:23AM -0700: > Since I expect to do a lot more reading than writing, this has me a > little worried! I'm getting this same slow read speed from the PHP and > Perl clients, and both talking to a local server on the BSD box or over > the network to the Linux box. What happens if you batch all the reads together, as one request? > Moving the Perl client from the 2 GHz Athlon (BSD) to the 266 MHz > Pentium II (Linux), oddly enough my speeds increase a bit: it takes a > mere 20 seconds to fetch 500 items (though the check + add sequence goes > up to 2 seconds). It doesn't seem to make a difference whether it's > talking to the local or the remote server. The slowdown must be due to something on the client side then. If you change the client's platform without changing the server's platform, the time changes from 50 to 20 seconds, right? -- avva From brion@pobox.com Sun Aug 10 13:18:53 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 05:18:53 -0700 Subject: Fast writes, slow reads In-Reply-To: <20030810143810.GA31414@pobox.com> References: <3F361F37.8010204@pobox.com> <20030810143810.GA31414@pobox.com> Message-ID: <3F36382D.7030000@pobox.com> Anatoly Vorobey wrote: >What happens if you batch all the reads together, as one request? > I'm having trouble getting this to work... so far, it takes the same amount of time (plus a fraction of a second for setting up more arrays) but I only actually get the last item requested returned in the hash. This _may_ be a problem in the PHP client, or I _may_ be doing something wrong. I haven't been able to get it working in perl, which _may_ be because I can't figure out how to get anything out of a hash reference. :) Here's the PHP code for the batched get: $keys = array(); for($i=0; $i<$max; $i++) { $keys[$i] = "test:intarray:$i"; } $res = $wgMemc->get_multi( $keys ); $arr = array(); for($i=0; $i<$max; $i++) { $n = $res[$key = "test:intarray:$i"]; if($n === NULL) { echo "
  • $i not in cache, adding\n"; $n = $i; $wgMemc->add( $key, $n ); } else { echo "
  • $n found\n"; } } >The slowdown must be due to something on the client side then. If you >change the client's platform without changing the server's platform, the >time changes from 50 to 20 seconds, right? > Seems to be so, yes. -- brion vibber (brion @ pobox.com) From lisa@gentoo.org Sun Aug 10 13:22:43 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: 10 Aug 2003 08:22:43 -0400 Subject: gentoo roadmap In-Reply-To: References: <1060370345.1765.30.camel@lisa.thedoh.com> <1060372622.1762.42.camel@lisa.thedoh.com> Message-ID: <1060518162.17210.0.camel@lisa.thedoh.com> --=-DU2pXl0ZLwquOQJ3qkpr Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Fri, 2003-08-08 at 17:12, Brad Fitzpatrick wrote: > In any case, the headers should be updated in Gentoo for the new system > call numbers, so programs like libevent can build with all their > functionality, even if the current running kernel doesn't support it. I'm not a real glibc guru, but the epoll.h header is shipped with our glibc. --=20 Regards, -Lisa --=-DU2pXl0ZLwquOQJ3qkpr Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/NjkSy0a1Vh5Jb8URArRuAKDB1XHhIlps8JVUBzTKk+SupTzixgCghHuH QnN+dkvL1P+S1hv4buKMHXw= =CsRO -----END PGP SIGNATURE----- --=-DU2pXl0ZLwquOQJ3qkpr-- From brad@danga.com Sun Aug 10 16:01:44 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 08:01:44 -0700 (PDT) Subject: Fast writes, slow reads In-Reply-To: <20030810104524.GA7244@danga.com> References: <3F361F37.8010204@pobox.com> <20030810104524.GA7244@danga.com> Message-ID: In particular, don't mismatch the wrong kernels with wrong libevents. Only use >= libevent-0.7a and epoll-lt (level-triggered). On Sun, 10 Aug 2003, Evan Martin wrote: > On Sun, Aug 10, 2003 at 03:32:23AM -0700, Brion Vibber wrote: > > The second run, which fetches back the set values, consistently takes > > about 50 seconds, about a tenth of a second per fetched item. 1000 items > > take 100 seconds, 50 items take 5 seconds. > > This sounds like a problem I was having related to the backend method > libevent was using. Which one are you using? To find out: > % EVENT_SHOW_METHOD=1 ./memcached > libevent using: select > > -- > Evan Martin > martine@danga.com > http://neugierig.org > > From brad@danga.com Sun Aug 10 16:05:33 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 08:05:33 -0700 (PDT) Subject: memcached in freebsd In-Reply-To: <3F36138E.50503@pobox.com> References: <3F36138E.50503@pobox.com> Message-ID: > On further investigation, I note two things: > > a) The "_load_items(): Failed to recieve valid header!" is what I get > anytime I ask for data that isn't available, and seems to be normal. Hm, that warning should be removed, then. Asking for data that's not there is a totally normal case. From brion@pobox.com Sun Aug 10 16:11:25 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 08:11:25 -0700 Subject: memcached in freebsd In-Reply-To: References: <3F36138E.50503@pobox.com> Message-ID: <3F36609D.8090102@pobox.com> Brad Fitzpatrick wrote: >>a) The "_load_items(): Failed to recieve valid header!" is what I get >>anytime I ask for data that isn't available, and seems to be normal. >> >> > >Hm, that warning should be removed, then. Asking for data that's not >there is a totally normal case. > It only shows that with debug output turned on. I suspect it's not meant to be as scary as it sounds. :) -- brion vibber (brion @ pobox.com) From martine@danga.com Sun Aug 10 19:50:46 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 11:50:46 -0700 Subject: [patch] increased verbosity setting Message-ID: <8D18F975-CB63-11D7-B1F0-003065B8A5CC@danga.com> --Apple-Mail-6-629867428 Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset=US-ASCII; format=flowed This patch lets you add multiple -v flags. Now -vv will dump the client commands/responses as they come in. (Someone on the list remarked that the -v output doesn't really show anything, but looking at the source indicates it shows errors. I was also surprised by seeing nothing when I added -v. Perhaps it would be better to show errors by default, use -v for this functionality I added, and either add a -q (quiet) or just suggest piping to /dev/null if you want no output.) --Apple-Mail-6-629867428 Content-Disposition: attachment; filename=memcached-verbose.patch Content-Transfer-Encoding: 7bit Content-Type: application/octet-stream; x-unix-mode=0644; name="memcached-verbose.patch" --- memcached/memcached.c Sun Aug 10 11:30:49 2003 +++ memcached-bsd/memcached.c Sun Aug 10 11:00:06 2003 @@ -208,6 +208,9 @@ void out_string(conn *c, char *str) { int len; + if (settings.verbose > 1) + fprintf(stderr, ">%d %s\n", c->sfd, str); + len = strlen(str); if (len + 2 > c->wsize) { /* ought to be always enough. just fail for simplicity */ @@ -458,6 +461,9 @@ * directly into it, then continue in nread_complete(). */ + if (settings.verbose > 1) + fprintf(stderr, "<%d %s\n", c->sfd, command); + if ((strncmp(command, "add ", 4) == 0 && (comm = NREAD_ADD)) || (strncmp(command, "set ", 4) == 0 && (comm = NREAD_SET)) || (strncmp(command, "replace ", 8) == 0 && (comm = NREAD_REPLACE))) { @@ -708,7 +714,7 @@ if (c->rbytes >= c->rsize) { char *new_rbuf = realloc(c->rbuf, c->rsize*2); if (!new_rbuf) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Couldn't realloc input buffer\n"); c->rbytes = 0; /* ignore what we read */ out_string(c, "SERVER_ERROR out of memory"); @@ -738,6 +744,7 @@ } int update_event(conn *c, int new_flags) { + printf("update_event %x %x\n", c->ev_flags, new_flags); if (c->ev_flags == new_flags) return 1; if (event_del(&c->event) == -1) return 0; @@ -757,7 +764,7 @@ int res; while (!exit) { - /*printf("state %d\n", c->state); */ + /* printf("state %d\n", c->state);*/ switch(c->state) { case conn_listening: addrlen = sizeof(addr); @@ -777,7 +784,7 @@ } newc = conn_new(sfd, conn_read, EV_READ | EV_PERSIST); if (!newc) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "couldn't create new connection\n"); close(sfd); return; @@ -794,7 +801,7 @@ } /* we have no command line and no data to read from network */ if (!update_event(c, EV_READ | EV_PERSIST)) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; @@ -835,7 +842,7 @@ } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_READ | EV_PERSIST)) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; @@ -844,7 +851,7 @@ break; } /* otherwise we have a real error, on which we close the connection */ - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Failed to read, and not due to blocking\n"); c->state = conn_closing; break; @@ -880,7 +887,7 @@ } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_READ | EV_PERSIST)) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; @@ -889,7 +896,7 @@ break; } /* otherwise we have a real error, on which we close the connection */ - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Failed to read, and not due to blocking\n"); c->state = conn_closing; break; @@ -913,7 +920,7 @@ } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_WRITE | EV_PERSIST)) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; @@ -923,7 +930,7 @@ } /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, we have a real error, on which we close the connection */ - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Failed to write, and not due to blocking\n"); c->state = conn_closing; break; @@ -947,7 +954,7 @@ } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_WRITE | EV_PERSIST)) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; @@ -957,7 +964,7 @@ } /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, we have a real error, on which we close the connection */ - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Failed to write, and not due to blocking\n"); c->state = conn_closing; break; @@ -1016,9 +1023,10 @@ c = (conn *)arg; c->which = which; + printf("event_handler %d\n", fd); /* sanity */ if (fd != c->sfd) { - if(settings.verbose) + if (settings.verbose > 0) fprintf(stderr, "Catastrophic: event fd doesn't match conn fd!\n"); conn_close(c); return; @@ -1126,6 +1134,7 @@ printf("-c max simultaneous connections, default is 1024\n"); printf("-k lock down all paged memory\n"); printf("-v verbose (print errors/warnings while in event loop)\n"); + printf("-vv more verbose (also print client commands/reponses)\n"); printf("-h print this help and exit\n"); printf("-i print memcached and libevent license\n"); return; @@ -1236,7 +1245,7 @@ lock_memory = 1; break; case 'v': - settings.verbose = 1; + settings.verbose++; break; case 'l': if (!inet_aton(optarg, &addr)) { --Apple-Mail-6-629867428-- From mellon@pobox.com Sun Aug 10 22:59:40 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Sun, 10 Aug 2003 21:59:40 +0000 Subject: [patch] increased verbosity setting In-Reply-To: <8D18F975-CB63-11D7-B1F0-003065B8A5CC@danga.com> References: <8D18F975-CB63-11D7-B1F0-003065B8A5CC@danga.com> Message-ID: <20030810215940.GA28898@pobox.com> On Sun, Aug 10, 2003 at 11:50:46AM -0700, Evan Martin wrote: > This patch lets you add multiple -v flags. Now -vv will dump the > client commands/responses as they come in. Does this patch have some debugging printf's mistakenly left in? + printf("update_event %x %x\n", c->ev_flags, new_flags); + printf("event_handler %d\n", fd); Also: - if(settings.verbose) + if (settings.verbose > 0) Why? ;) From martine@danga.com Sun Aug 10 20:06:37 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 12:06:37 -0700 Subject: [test case] terrible performance with select() Message-ID: --Apple-Mail-12-630818559 Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset=US-ASCII; format=flowed As a few people (including me) have found out, memcache is suspiciously slow when using libevent backends other than epoll. (I think? I guess I've only seen the behavior myself with select().) FWIW, it appears the problem that event_hander is taking up to five seconds to get called, but only after repeat commands. Why that happens is a more difficult problem, because it works fine with epoll so it must be some behavior that's inconsistent underneath libevent. Attached is a test case that exhibits the behavior. Of note is the perfectly round numbers that come up when it's being slow. Is something timing out here? (The "delay" here is the delay between subsequent requests.) lulu:~/projects/memcached/api/python% ./slow.py 1 Trying get()s with 1.00s delay get took 0.13 seconds get took 0.30 seconds get took 4.00 seconds get took 4.00 seconds get took 4.00 seconds get took 4.00 seconds lulu:~/projects/memcached/api/python% ./slow.py 0.9 Trying get()s with 0.90s delay get took 0.18 seconds get took 1.38 seconds get took 4.10 seconds get took 4.10 seconds get took 4.10 seconds get took 4.10 seconds --Apple-Mail-12-630818559 Content-Disposition: attachment; filename=slow.py Content-Transfer-Encoding: 7bit Content-Type: application/octet-stream; x-unix-mode=0755; name="slow.py" #!/usr/bin/env python import sys import memcache import time sleeptime = 1 if len(sys.argv) > 1: sleeptime = float(sys.argv[1]) mc = memcache.Client(['127.0.0.1:11211']) print "Trying get()s with %0.2fs delay" % sleeptime try: while 1: t1 = time.time() mc.get("hi") t2 = time.time() print "get took %0.2f seconds" % (t2-t1) time.sleep(sleeptime) except KeyboardInterrupt: pass --Apple-Mail-12-630818559-- From martine@danga.com Sun Aug 10 20:14:14 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 12:14:14 -0700 Subject: [patch] increased verbosity setting In-Reply-To: <20030810215940.GA28898@pobox.com> Message-ID: On Sunday, August 10, 2003, at 02:59 PM, Anatoly Vorobey wrote: > On Sun, Aug 10, 2003 at 11:50:46AM -0700, Evan Martin wrote: >> This patch lets you add multiple -v flags. Now -vv will dump the >> client commands/responses as they come in. > > Does this patch have some debugging printf's mistakenly left in? > > + printf("update_event %x %x\n", c->ev_flags, new_flags); > + printf("event_handler %d\n", fd); Oops. :) I noticed that just after I sent it, and then sent a new patch that also included a ChangeLog entry, but I managed to send that new patch only to myself. (I'm using the OSX mail client and I'm a bit unfamiliar with it, which is also why these patches are coming out a bit weird.) > Also: > > - if(settings.verbose) > + if (settings.verbose > 0) > > Why? ;) Yeah, good point. I had something else in mind when I made the change... I guess it'd be most appropriate to do some sort of enum { LOG_ERRORS, LOG_PROTOCOL }; and then use if (settings.verbose > LOG_ERRORS) instead of that > 0, but that's probably not worth the effort. From martine@danga.com Sun Aug 10 20:22:08 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 12:22:08 -0700 Subject: [patch] python api touchup Message-ID: --Apple-Mail-14-631750345 Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset=US-ASCII; format=flowed Most importantly: I renamed a class and missed a place where it was referenced, which means it dies inappropriately if a host goes down. --Apple-Mail-14-631750345 Content-Disposition: attachment; filename=memcached-python.patch Content-Transfer-Encoding: 7bit Content-Type: application/octet-stream; x-unix-mode=0644; name="memcached-python.patch" diff -ur --exclude=CVS memcached-bsd/api/python/ChangeLog memcached-pyfix/api/python/ChangeLog --- memcached-bsd/api/python/ChangeLog Sun Aug 10 12:16:41 2003 +++ memcached-pyfix/api/python/ChangeLog Sun Aug 10 12:20:17 2003 @@ -1,3 +1,9 @@ +Thu, 10 Aug 2003 12:17:50 -0700 Evan Martin + + * Slightly more verbose self-test output. + * Fix mark_dead() to use proper classname. + * Make pooltest.py run from the test directory. + Thu, 07 Aug 2003 16:32:32 -0700 Evan Martin * Add incr, decr, and delete. diff -ur --exclude=CVS memcached-bsd/api/python/memcache.py memcached-pyfix/api/python/memcache.py --- memcached-bsd/api/python/memcache.py Sun Aug 10 12:16:44 2003 +++ memcached-pyfix/api/python/memcache.py Sun Aug 10 12:15:50 2003 @@ -445,7 +445,7 @@ def mark_dead(self, reason): print "MemCache: %s: %s. Marking dead." % (self, reason) - self.deaduntil = time.time() + Host._DEAD_RETRY + self.deaduntil = time.time() + _Host._DEAD_RETRY self.close_socket() def _get_socket(self): @@ -508,7 +508,7 @@ d = " (dead until %d)" % self.deaduntil return "%s:%d%s" % (self.ip, self.port, d) -def _test(): +def _doctest(): import doctest, memcache servers = ["127.0.0.1:11211"] mc = Client(servers, debug=1) @@ -516,7 +516,8 @@ return doctest.testmod(memcache, globs=globs) if __name__ == "__main__": - _test() + print "Testing docstrings..." + _doctest() print "Running tests:" print #servers = ["127.0.0.1:11211", "127.0.0.1:11212"] Only in memcached-pyfix/api/python: memcache.pyc Only in memcached-pyfix/api/python: slow.py diff -ur --exclude=CVS memcached-bsd/api/python/tests/pooltest.py memcached-pyfix/api/python/tests/pooltest.py --- memcached-bsd/api/python/tests/pooltest.py Sun Aug 10 12:16:44 2003 +++ memcached-pyfix/api/python/tests/pooltest.py Sun Aug 10 12:15:50 2003 @@ -5,6 +5,8 @@ Bring up two memcaches on :11211 and :11212. Try killing one or both. If this code raises any exceptions, it's a bug.""" +import sys +sys.path.append("..") import memcache import time --Apple-Mail-14-631750345-- From brad@danga.com Sun Aug 10 23:30:10 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 15:30:10 -0700 (PDT) Subject: [patch] python api touchup In-Reply-To: References: Message-ID: Cool. Incremented the version number to 1.2 and put on the website. On Sun, 10 Aug 2003, Evan Martin wrote: > Most importantly: I renamed a class and missed a place where it was > referenced, which means it dies inappropriately if a host goes down. > > From brad@danga.com Sun Aug 10 23:34:35 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 15:34:35 -0700 (PDT) Subject: rtsig.patch (fwd) Message-ID: This message is in MIME format. The first part should be readable text, while the remaining parts are likely unreadable without MIME-aware tools. Send mail to mime@docserver.cac.washington.edu for more info. --yrj/dFKFPuw6o+aM Content-Type: TEXT/PLAIN; CHARSET=US-ASCII Content-ID: Content-Disposition: INLINE Taral from LiveJournal added realtime signal support to libevent, which is supported in Linux 2.4 natively (without patches like epoll) and is way better than select/poll. I haven't tested this, but it might be interesting to somebody else on this list. - Brad --yrj/dFKFPuw6o+aM Content-Type: TEXT/PLAIN; CHARSET=us-ascii Content-Transfer-Encoding: QUOTED-PRINTABLE Content-ID: Content-Description: Content-Disposition: ATTACHMENT; FILENAME="rtsig.patch" -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.1 (GNU/Linux) iD8DBQE+80SQoQQF8xCPwJQRAo/BAJ97/a+7qSEHkAwk3RuMTZ+11UhbMgCfTd1R nFN3sTRxzxhOcWnnVy39PyQ= =orNr -----END PGP SIGNATURE----- --yrj/dFKFPuw6o+aM-- From brad@danga.com Sun Aug 10 23:52:47 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 15:52:47 -0700 (PDT) Subject: rtsig.patch (fwd) In-Reply-To: <20030811014057.GA30166@pobox.com> References: <20030811014057.GA30166@pobox.com> Message-ID: Er, let's put it here then: http://www.danga.com/memcached/dist/rtsig.patch - Brad On Mon, 11 Aug 2003, Anatoly Vorobey wrote: > You wrote on Sun, Aug 10, 2003 at 03:34:35PM -0700: > > Taral from LiveJournal added realtime signal support to > > libevent, which is supported in Linux 2.4 natively (without patches like > > epoll) and is way better than select/poll. > > > > I haven't tested this, but it might be interesting to somebody else on > > this list. > > You probably realized already, but there's no patch, only a GPG sig. > > -- > avva > > From brad@danga.com Mon Aug 11 00:16:37 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 16:16:37 -0700 (PDT) Subject: PHP API 1.0.7 In-Reply-To: <3F35EB0B.7010506@pobox.com> References: <3F352A7D.9090306@gilfether.com> <3F35EB0B.7010506@pobox.com> Message-ID: PHP client version 1.0.7 is on the website, with your debugging abstraction merged by hand (it didn't apply against Ryan's 1.0.7). Because I did it by hand and didn't test, it may have syntax errors. Let me know. I now have a release script for PHP, so it's easy for me to make new releases. On Sat, 9 Aug 2003, Brion Vibber wrote: > Ryan Gilfether wrote: > > > A new version of the PHP API has been completed, verson 1.0.7. > > > Cool! Where can I find it? > > I'm doing some preliminary work on getting memcached support into > Wikipedia (or rather MediaWiki, as we've recently redubbed the software > thereof), so far I've started with the 1.0.6 PHP client. > > > Along with that 3 new function have been added > > error() - returns an error code for the last error genterated > > error_string() - returns a string with a description of the error > > error_clear() - clears the last error > > > On a related note, the debug mode doesn't mix well with MediaWiki's > output model. I replaced all the "print"s with calls to a _debug() > private function which can I can then override in a subclass to dump > things to our debug log; diff for 1.0.6 attached. > > -- brion vibber (brion @ pobox.com) > From brad@danga.com Mon Aug 11 00:30:17 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 16:30:17 -0700 (PDT) Subject: PHP client's _load_items (was: memcached in freebsd) In-Reply-To: <20030810131120.GA30897@pobox.com> References: <3F36138E.50503@pobox.com> <20030810131120.GA30897@pobox.com> Message-ID: Yeah, the _load_items function in the PHP client is kinda scary. There's no way to treat a socket like a stream in PHP? The command sending in _load_items is good, but the parsing (especially not respecting END\r\n on any "line") is wrong. It'd be so much easier if you could read a line or a block at a time. Then you wouldn't have to remember so much state. Unrelated, that file's in need of some tabs vs. spaces cleanup. Go one way or the other and be consistent, or at least add all the emacs/vim hints at the top/bottom, so people's editors know how wide you thought your spaces were. Ryan, in the future feel free to post diffs to the list, rather than tarballs to me directly, then everybody can get access to things when I'm away for a few hours, and it's easier to read a diff, too. If you need help with cvs and/or making diffs from cvs, email me off-list. - Brad On Sun, 10 Aug 2003, Anatoly Vorobey wrote: > Brion Vibber wrote on Sun, Aug 10, 2003 at 02:42:38AM -0700: > > On further investigation, I note two things: > > > > a) The "_load_items(): Failed to recieve valid header!" is what I get > > anytime I ask for data that isn't available, and seems to be normal. > > Well, if some data you asked for isn't available, it's _not_ an error, > and you don't fail to receive a valid header, so something strange is > going on. A cursory look at the client's source (I don't really know any > PHP) seems to indicate that the client makes some unwarranted > assumptions. I _think_ it only recognises the "END\r\n" line which > finishes any batch of data a server sends when this line comes in the > same buffer (as a result of the same read call) with some previous data > for some key. As a result, when the server fails to match any keys, and > only sends "END\r\n" (correctly, according to the protocol), the client > thinks it's an error. This by itself may be harmless, but I think that > more serious errors could occur if the "END\r\n" doesn't come in the > same IP packet as the rest of server's reply (say if the reply is too > large to fit in one packet) and subsequently arrives via a separate > read request. This logic should be fixed (there shouldn't be any > assumptions about how data is separated across read/write calls). > > > D'oh! Looks like relative expiration times are a new feature of > > 1.1.7/1.1.8, and 1.1.6 expects either 0 or a full Unix timestamp. > > That's correct (don't use 1.1.7, it has an embarrassing bug related to > this very feature). > > --avva > > From brad@danga.com Mon Aug 11 00:39:18 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 16:39:18 -0700 (PDT) Subject: Fast writes, slow reads Message-ID: Brion, Can you reproduce this slow behavior with the Linux x86 binary at http://www.danga.com/memcached/dist/binaries/linux-x86/ ? I want to rule out any sort of build differences, since the binary above is what we use on LiveJournal. (and it flies for us...) If it's still slow with that binary and with the Perl client, it's definitely time to point fingers at libevent. Nothing I see in the PHP client should make things slow, just incorrectly warning in some cases. If you can, make yourself a new kernel. Here's the latest patch, against 2.4.21: http://www.xmailserver.org/linux-patches/epoll-lt-2.4.21-0.16.diff - Brad On Sun, 10 Aug 2003, Brion Vibber wrote: > I'm finding that reading data back from memcached is surprisingly slow. > I've probably done something wrong. :) > > I wrote a simple loop that reads 500 integers or strings, adding each > one to the cache if it's not already in. The first run, which checks for > 500 uncached items and adds 500 items to the cache, runs fast enough; > more or less a second. > > The second run, which fetches back the set values, consistently takes > about 50 seconds, about a tenth of a second per fetched item. 1000 items > take 100 seconds, 50 items take 5 seconds. > > Since I expect to do a lot more reading than writing, this has me a > little worried! I'm getting this same slow read speed from the PHP and > Perl clients, and both talking to a local server on the BSD box or over > the network to the Linux box. > > Moving the Perl client from the 2 GHz Athlon (BSD) to the 266 MHz > Pentium II (Linux), oddly enough my speeds increase a bit: it takes a > mere 20 seconds to fetch 500 items (though the check + add sequence goes > up to 2 seconds). It doesn't seem to make a difference whether it's > talking to the local or the remote server. > > Any ideas? I've attached the Perl version of the test script; I'm using > the 1.0.6 MemCachedClient... Perl 5.6.1 is installed on the FreeBSD box, > 5.8.0 on the Linux box. > > -- brion vibber (brion @ pobox.com) > From brad@danga.com Mon Aug 11 00:43:36 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 16:43:36 -0700 (PDT) Subject: Fast writes, slow reads In-Reply-To: <3F3623E7.5040105@pobox.com> References: <3F361F37.8010204@pobox.com> <20030810104524.GA7244@danga.com> <3F3623E7.5040105@pobox.com> Message-ID: It's even slow when you're using kqueue?! That's supposed to be [slightly] faster than epoll. Was it slow with kqueue and the Perl client or the PHP client? One problem we run into every couple months at LiveJournal is new network hardware coming up in the wrong duplex. Auto-negotiation never seems to work that well. Can you verify both sides are in the same duplex, either half or full? - Brad On Sun, 10 Aug 2003, Brion Vibber wrote: > Evan Martin wrote: > > >This sounds like a problem I was having related to the backend method > >libevent was using. > > > Uh-oh... :) > > > Which one are you using? To find out: > >% EVENT_SHOW_METHOD=1 ./memcached > >libevent using: select > > > FreeBSD 5.1, libevent 0.7a installed from ports: > $ EVENT_SHOW_METHOD=1 memcached > libevent using: kqueue > > Red Hat 8.0, libevent 0.7a installed from source with no special options: > $ EVENT_SHOW_METHOD=1 ./memcached > libevent using: poll > > -- brion vibber (brion @ pobox.com) > > From brion@pobox.com Mon Aug 11 01:47:35 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 17:47:35 -0700 Subject: Fast writes, slow reads In-Reply-To: References: <3F361F37.8010204@pobox.com> <20030810104524.GA7244@danga.com> <3F3623E7.5040105@pobox.com> Message-ID: <3F36E7A7.5000902@pobox.com> Brad Fitzpatrick wrote: >It's even slow when you're using kqueue?! That's supposed to be >[slightly] faster than epoll. > >Was it slow with kqueue and the Perl client or the PHP client? > I have tried combinations of the following: ports-patched 1.1.6 server on FreeBSD, using kqueue self-compiled 1.1.8 server on Linux, using poll dl'd binary 1.1.8 server on Linux, using poll (I haven't yet managed to get a kernel recompiled with the epoll patch.) with PHP client w/ PHP 4.3.2 on BSD Perl client w/ Perl 5.6.1 on BSD Perl client w/ Perl 5.8.0 on Linux Python client w/ Python 2.2.1 on Linux Python client w/ Python 2.2.2 on BSD So far I've consistently found that: * gets are reasonably fast when there's no data to return * sets, adds, replaces, and deletes are reasonably fast, whether or not the key already exists * gets have a large fixed overhead when there is data to return, which is consistent across all three client libraries and server connections but varies according to the client's system (0.10 seconds on 2GHz Athlon/FreeBSD, 0.04 seconds on 266MHz PII/Linux) With Evan's "slow.py" demo, I see the wait disappear after the first get when using a delay longer than the minimum time on the Linux box, connecting to either server: $ python slow.py 0.03 Trying get()s with 0.03s delay get took 0.04 seconds get took 0.04 seconds get took 0.04 seconds ... $ python slow.py 0.05 Trying get()s with 0.05s delay get took 0.04 seconds get took 0.00 seconds get took 0.00 seconds ... But it never goes away on the BSD box, connecting to either server: $ python slow.py 1 Trying get()s with 1.00s delay get took 0.10 seconds get took 0.10 seconds get took 0.10 seconds ... >One problem we run into every couple months at LiveJournal is new >network hardware coming up in the wrong duplex. Auto-negotiation never >seems to work that well. Can you verify both sides are in the same >duplex, either half or full? > I get the same performance (within my testing tolerance, anyway) over loopback as I do when talking to the other box over the network... -- brion vibber (brion @ pobox.com) From brion@pobox.com Mon Aug 11 02:36:45 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 18:36:45 -0700 Subject: Fast writes, slow reads In-Reply-To: <3F36E7A7.5000902@pobox.com> References: <3F361F37.8010204@pobox.com> <20030810104524.GA7244@danga.com> <3F3623E7.5040105@pobox.com> <3F36E7A7.5000902@pobox.com> Message-ID: <3F36F32D.70308@pobox.com> I wrote: > With Evan's "slow.py" demo, I see the wait disappear after the first > get when using a delay longer than the minimum time on the Linux box, > connecting to either server: [snip] > $ python slow.py 0.05 > Trying get()s with 0.05s delay > get took 0.04 seconds > get took 0.00 seconds > get took 0.00 seconds > ... If I modify the script to use a different key (for which there is data) for each request, the slowdown remains consistent does not go away with increased delays between requests. -- brion vibber (brion @ pobox.com) From brad@danga.com Mon Aug 11 03:54:57 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 19:54:57 -0700 (PDT) Subject: Read vs. write performance (sloppiness in memcache) Message-ID: I've been investigating the findings that writes are faster than reads. I did a bunch of network sniffing and found the problem is how memcached returns results. It sends way too many packets, resulting in tons of round-trips. In the first case, where the cache is empty, and the client sets 500 keys, it does a get (one packet), and the server sends back a reply saying it has nothing (one packet), the client sets it (one packet), and the server says okay (one packet). Every packet in the transaction except the setup and teardown is a PSH+ACK. That is, it's sending new data and acknowledging the data it just got. However, memcached is a pig when it comes to sending results. Client: get foo, bar Server: I have foo! Client: Okay, I heard you. I'm bored. (ACK) Server: Here is foo. Client: Okay. I'm still waiting (ACK) Server: I have bar! Client: *yawn* (ACK) Server: Here is bar. And we're done. Client: Okay. (ACK) It should be: Client: get foo, bar (PSH) Server: I have foo and bar. here they are. we're done. (ACK,PSH) Client: Okay. (ACK) We'll fix this. In the meantime, don't let this deter you from using memcache. LiveJournal is running with this "slow" code and it's still ridiculously fast (and way faster than a database). This finding just means we can go even faster. - Brad From mellon@pobox.com Mon Aug 11 07:02:56 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Mon, 11 Aug 2003 06:02:56 +0000 Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: References: Message-ID: <20030811060256.GA7617@pobox.com> You wrote on Sun, Aug 10, 2003 at 07:54:57PM -0700: > I've been investigating the findings that writes are faster than reads. Here's someone that seems to've run into the same problem. He even reports better results when client is switched from FreeBSD to Linux (just as Brion wrote): http://www.openldap.org/lists/openldap-devel/199907/msg00079.html -- avva From corvus@vadept.com Mon Aug 11 04:13:45 2003 From: corvus@vadept.com (Frank Precissi) Date: Sun, 10 Aug 2003 20:13:45 -0700 Subject: Benchmark? Message-ID: <200308102013.45503.corvus@vadept.com> Hi everyone, Is there a standardized benchmark/test harness available for memcache? With all of the different libevent options (select,kqueue,poll,etc) and different programming/os platforms, I was curious if there was a testing standard developed (insert 500 items, pull a timed random 20 items). We could publish benchmark results based upon libevent/os/language so admin's who use memcache have some sort of reference point to determine if they compiled it correctly or if they are using the slowest libevent for their specific OS. Just a thought.. :) Frank -- A man's gotta know his limitations. -- Clint Eastwood, "Dirty Harry" From mellon@pobox.com Mon Aug 11 07:27:55 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Mon, 11 Aug 2003 06:27:55 +0000 Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: References: Message-ID: <20030811062755.GA7772@pobox.com> You wrote on Sun, Aug 10, 2003 at 07:54:57PM -0700: > It should be: > > Client: get foo, bar (PSH) > Server: I have foo and bar. here they are. we're done. (ACK,PSH) > Client: Okay. (ACK) Nah. The problem is not many packets, but the fact that the TCP/IP stack waits a little bit on each of them before sending, hoping we're going to send another one... but when the client requests 500 values one after the other, as in Brion's test, it never does: client: give me key1 server: sending value1 OS: [let's wait a bit] waiting... OS: ok, nothing more has come. sending value1 client: received value1 client: give me key2 .... By setting TCP_NODELAY on the server socket, we eliminate the waits and seem to completely fix the problem, at least for this particular kind of test, where we don't have many consecutive writes. The wait time for receiving 500 values drops from 20sec. to 1 or 0. Brion, can you verify this with your setup? The patch below makes the server set TCP_NODELAY. On the other hand, what if the client requests 100 keys in one GET request? It appears to me that until now, we would send as few IP packets as possible, because we didn't have TCP_NODELAY set and so the OS was waiting a few dozens of milliseconds for us to send more, and we always send more very fast (we don't let go of a GET request as long as the OS accepts our write()'s synchronously). With TCP_NODELAY set, we will instead of have 200 packets instead of, say, 20. Perhaps we should run this under a sniffer, both with and without TCP_NODELAY, to figure out whether this is indeed true, and how this options affects performance of large multi-key GET requests. If it affects the performance badly, but we still want TCP_NODELAY to boost the performance of many consecutive single-key requests, we can think of buffering on the server side, before sending out write()'s -- since this may drastically affect our memory usage, I'd rather not do it unless we really have to. main -> wcmtools src/memcached/memcached.c --- cvs/wcmtools/memcached/memcached.c Tue Jul 29 22:53:49 2003 +++ src/memcached/memcached.c Sun Aug 10 20:13:20 2003 @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -1052,6 +1053,7 @@ setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof(flags)); setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, &flags, sizeof(flags)); setsockopt(sfd, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)); + setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(flags)); addr.sin_family = AF_INET; addr.sin_port = htons(port); -- avva From mellon@pobox.com Mon Aug 11 07:30:34 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Mon, 11 Aug 2003 06:30:34 +0000 Subject: Benchmark? In-Reply-To: <200308102013.45503.corvus@vadept.com> References: <200308102013.45503.corvus@vadept.com> Message-ID: <20030811063034.GA7788@pobox.com> On Sun, Aug 10, 2003 at 08:13:45PM -0700, Frank Precissi wrote: > Hi everyone, > > Is there a standardized benchmark/test harness available for memcache? With > all of the different libevent options (select,kqueue,poll,etc) and different > programming/os platforms, I was curious if there was a testing standard > developed (insert 500 items, pull a timed random 20 items). Not yet, but there should be one; moreover, it should measure not just saves and single-item pulls (like Brion's little test does), but also multiple-item pulls with different amounts of items per request. It seems we'll run into some trade-offs in this area anyway, so we might as well collect more info about it. -- avva sorely in need of a memcache for my brain From brion@pobox.com Mon Aug 11 04:55:00 2003 From: brion@pobox.com (Brion Vibber) Date: Sun, 10 Aug 2003 20:55:00 -0700 Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: <20030811062755.GA7772@pobox.com> References: <20030811062755.GA7772@pobox.com> Message-ID: <3F371394.8090607@pobox.com> Anatoly Vorobey wrote: >By setting TCP_NODELAY on the server socket, we eliminate the waits >and seem to completely fix the problem, at least for this particular >kind of test, where we don't have many consecutive writes. The wait time >for receiving 500 values drops from 20sec. to 1 or 0. Brion, can you >verify this with your setup? The patch below makes the server set >TCP_NODELAY. > Verified. With the patched server, it runs like lightning. Tested both server and client on both BSD and Linux. >On the other hand, what if the client requests 100 keys in one GET >request? > Can someone whip up a quick script that tests this? I've been unable to make it work using either the Perl or PHP clients, but I may just not know what I'm doing. -- brion vibber (brion @ pobox.com) From brad@danga.com Mon Aug 11 04:56:17 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 20:56:17 -0700 (PDT) Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: <20030811062755.GA7772@pobox.com> References: <20030811062755.GA7772@pobox.com> Message-ID: Yeah, this patch makes a world of difference. Instead of 20 seconds, it now takes less than 1. However, it's even more packet-happy than before. Dude, we have a more-than-plentiful per-connect buffer on the server side (DATA_BUFFER_SIZE = 2k, by default). Let's use as much of it as we can before writing. I'm not advocating growing it to fit more, but if something else has already grown it (quite likely, since we grow it whenever sending large values), let's write into it, keep track of where we're at (and thus how much we have remaining), and as long as our output can keep fitting in our available per-connection memory, let's put it there until we're out of space, or until we're done. Looking at this tcpdump, we're doing: One packet for "VALUE test:intarray:0 0 1\r\n" One packet for "0\r\n" One packet for "END\r\n" Super lame. That'd fit in 2k. On Mon, 11 Aug 2003, Anatoly Vorobey wrote: > You wrote on Sun, Aug 10, 2003 at 07:54:57PM -0700: > > It should be: > > > > Client: get foo, bar (PSH) > > Server: I have foo and bar. here they are. we're done. (ACK,PSH) > > Client: Okay. (ACK) > > Nah. The problem is not many packets, but the fact that the TCP/IP > stack waits a little bit on each of them before sending, hoping we're > going to send another one... but when the client requests 500 values > one after the other, as in Brion's test, it never does: > > client: give me key1 > server: sending value1 > OS: [let's wait a bit] > waiting... > OS: ok, nothing more has come. sending value1 > client: received value1 > client: give me key2 > .... > > By setting TCP_NODELAY on the server socket, we eliminate the waits > and seem to completely fix the problem, at least for this particular > kind of test, where we don't have many consecutive writes. The wait time > for receiving 500 values drops from 20sec. to 1 or 0. Brion, can you > verify this with your setup? The patch below makes the server set > TCP_NODELAY. > > On the other hand, what if the client requests 100 keys in one GET > request? It appears to me that until now, we would send as few IP > packets as possible, because we didn't have TCP_NODELAY set and so the > OS was waiting a few dozens of milliseconds for us to send more, and > we always send more very fast (we don't let go of a GET request as long > as the OS accepts our write()'s synchronously). With TCP_NODELAY set, > we will instead of have 200 packets instead of, say, 20. Perhaps we > should run this under a sniffer, both with and without TCP_NODELAY, to > figure out whether this is indeed true, and how this options affects > performance of large multi-key GET requests. If it affects the > performance badly, but we still want TCP_NODELAY to boost the > performance of many consecutive single-key requests, we can think of > buffering on the server side, before sending out write()'s -- since this > may drastically affect our memory usage, I'd rather not do it unless we > really have to. > > > > main -> wcmtools src/memcached/memcached.c > --- cvs/wcmtools/memcached/memcached.c Tue Jul 29 22:53:49 2003 > +++ src/memcached/memcached.c Sun Aug 10 20:13:20 2003 > @@ -30,6 +30,7 @@ > #include > #include > #include > +#include > #include > #include > #include > @@ -1052,6 +1053,7 @@ > setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof(flags)); > setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, &flags, sizeof(flags)); > setsockopt(sfd, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)); > + setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(flags)); > > addr.sin_family = AF_INET; > addr.sin_port = htons(port); > > -- > avva > > From brad@danga.com Mon Aug 11 04:59:47 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 20:59:47 -0700 (PDT) Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: <3F371394.8090607@pobox.com> References: <20030811062755.GA7772@pobox.com> <3F371394.8090607@pobox.com> Message-ID: On Sun, 10 Aug 2003, Brion Vibber wrote: > Anatoly Vorobey wrote: > > >On the other hand, what if the client requests 100 keys in one GET > >request? > > > > Can someone whip up a quick script that tests this? I've been unable to > make it work using either the Perl or PHP clients, but I may just not > know what I'm doing. > > -- brion vibber (brion @ pobox.com) It'd be fast, but still packet heavy. We know the real issue now. We should probably disable Nagel in the clients, too, or do explicit flushes at certain points. Actually, I already had flushes in my Perl client.... but it looks like the PHP and Python clients could benefit from turning Nagel off (PHP), or doing flushes (Python), based on their output methods. - Brad From lisa@gentoo.org Mon Aug 11 05:02:24 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: 11 Aug 2003 00:02:24 -0400 Subject: PHP API 1.0.7 In-Reply-To: <3F352A7D.9090306@gilfether.com> References: <3F352A7D.9090306@gilfether.com> Message-ID: <1060574543.1832.9.camel@lisa.thedoh.com> --=-J5kGgli/oV+/7kwD30AS Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: > A new version of the PHP API has been completed, verson 1.0.7. > I've had some requests for adding PHPDocs style comments to the API.=20 > Along with that 3 new function have been added > error() - returns an error code for the last error genterated > error_string() - returns a string with a description of the error > error_clear() - clears the last error >=20 > Ryan Not to be incredibly anal, but the .php file should not have any newlines after the closing ?> tag -- this tends to mess with headers should it be included after headers have been sent. :( --=20 Regards, -Lisa --=-J5kGgli/oV+/7kwD30AS Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/NxVPy0a1Vh5Jb8URApoXAJ942gJ6KbdliQH2K67onIROzJMEVACfSP8H bWwY1IqWWItsKdJBp/OLwd0= =QsM/ -----END PGP SIGNATURE----- --=-J5kGgli/oV+/7kwD30AS-- From brad@danga.com Mon Aug 11 05:07:31 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 21:07:31 -0700 (PDT) Subject: PHP API 1.0.7 In-Reply-To: <1060574543.1832.9.camel@lisa.thedoh.com> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> Message-ID: PHP doesn't detect that? Sad. Ryan might've sent me it corrected and maybe I messed it up when I was merging the other patch. I'll make sure future releases have no bytes after the "?>". - Brad On Sun, 11 Aug 2003, Lisa Marie Seelye wrote: > On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: > > A new version of the PHP API has been completed, verson 1.0.7. > > I've had some requests for adding PHPDocs style comments to the API. > > Along with that 3 new function have been added > > error() - returns an error code for the last error genterated > > error_string() - returns a string with a description of the error > > error_clear() - clears the last error > > > > Ryan > > Not to be incredibly anal, but the .php file should not have any > newlines after the closing ?> tag -- this tends to mess with headers > should it be included after headers have been sent. :( > > > -- > Regards, > -Lisa > > From brad@danga.com Mon Aug 11 05:31:00 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 21:31:00 -0700 (PDT) Subject: PHP client's _load_items In-Reply-To: <3F3704B0.1010903@gilfether.com> References: <3F36138E.50503@pobox.com> <20030810131120.GA30897@pobox.com> <3F3704B0.1010903@gilfether.com> Message-ID: PHP client 1.0.8 is on the website now. Mostly whitespace/punctuation cleanup. Details below. Lisa, I removed the trailing whitespace so PHP won't send HTTP headers prematurely. (Ryan, you'll have to do that on your side too.) I look forward to a proper fix to _load_items. - Brad On Sun, 10 Aug 2003, Ryan Gilfether wrote: > Brad > > attached is the patch to move the 1.0.7 php api to 1.0.8. it fixes the > incorrect debug wording Brion mentioned earlier and also incorporates > his patch that for the debugging abstraction. It also fixes some of the > formatting problems, my editor seemed to use spaces where tabs should > have been applied in some areas. I might consider editing on emacs or vi > in the future. > > I'll work on the _load_items() function for the next release and see > what I can do about it. > > Ryan > > Brad Fitzpatrick wrote: > > Yeah, the _load_items function in the PHP client is kinda scary. > > > > There's no way to treat a socket like a stream in PHP? The command > > sending in _load_items is good, but the parsing (especially not > > respecting END\r\n on any "line") is wrong. It'd be so much easier if > > you could read a line or a block at a time. Then you wouldn't have > > to remember so much state. > > > > Unrelated, that file's in need of some tabs vs. spaces cleanup. Go one > > way or the other and be consistent, or at least add all the emacs/vim > > hints at the top/bottom, so people's editors know how wide you thought > > your spaces were. > > > > Ryan, in the future feel free to post diffs to the list, rather than > > tarballs to me directly, then everybody can get access to things when I'm > > away for a few hours, and it's easier to read a diff, too. If you need > > help with cvs and/or making diffs from cvs, email me off-list. > > > > - Brad > > > > > > On Sun, 10 Aug 2003, Anatoly Vorobey wrote: > > > > > >>Brion Vibber wrote on Sun, Aug 10, 2003 at 02:42:38AM -0700: > >> > >>>On further investigation, I note two things: > >>> > >>>a) The "_load_items(): Failed to recieve valid header!" is what I get > >>>anytime I ask for data that isn't available, and seems to be normal. > >> > >>Well, if some data you asked for isn't available, it's _not_ an error, > >>and you don't fail to receive a valid header, so something strange is > >>going on. A cursory look at the client's source (I don't really know any > >>PHP) seems to indicate that the client makes some unwarranted > >>assumptions. I _think_ it only recognises the "END\r\n" line which > >>finishes any batch of data a server sends when this line comes in the > >>same buffer (as a result of the same read call) with some previous data > >>for some key. As a result, when the server fails to match any keys, and > >>only sends "END\r\n" (correctly, according to the protocol), the client > >>thinks it's an error. This by itself may be harmless, but I think that > >>more serious errors could occur if the "END\r\n" doesn't come in the > >>same IP packet as the rest of server's reply (say if the reply is too > >>large to fit in one packet) and subsequently arrives via a separate > >>read request. This logic should be fixed (there shouldn't be any > >>assumptions about how data is separated across read/write calls). > >> > >> > >>>D'oh! Looks like relative expiration times are a new feature of > >>>1.1.7/1.1.8, and 1.1.6 expects either 0 or a full Unix timestamp. > >> > >>That's correct (don't use 1.1.7, it has an embarrassing bug related to > >>this very feature). > >> > >>--avva > >> > >> > > > > > > > > -- > Ryan Gilfether > Software Engineer > http://www.gilfether.com > From martine@danga.com Mon Aug 11 05:34:59 2003 From: martine@danga.com (Evan Martin) Date: Sun, 10 Aug 2003 21:34:59 -0700 Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: References: <20030811062755.GA7772@pobox.com> <3F371394.8090607@pobox.com> Message-ID: <20030811043459.GA28192@danga.com> On Sun, Aug 10, 2003 at 08:59:47PM -0700, Brad Fitzpatrick wrote: > Actually, I already had flushes in my Perl client.... but it looks like > the PHP and Python clients could benefit from turning Nagel off (PHP), or > doing flushes (Python), based on their output methods. I believe the send_all function used in the Python code flushes. -- Evan Martin martine@danga.com http://neugierig.org From ryan@gilfether.com Mon Aug 11 05:37:40 2003 From: ryan@gilfether.com (Ryan Gilfether) Date: Mon, 11 Aug 2003 00:37:40 -0400 Subject: PHP API 1.0.7 In-Reply-To: <1060574543.1832.9.camel@lisa.thedoh.com> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> Message-ID: <3F371D94.5040100@gilfether.com> yes, you're right, I didnt notice that a newline got included at the end of the file. I'll be sure to remove it in future releases. Thanks for pointing it out. Ryan Lisa Marie Seelye wrote: > On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: > >>A new version of the PHP API has been completed, verson 1.0.7. >>I've had some requests for adding PHPDocs style comments to the API. >>Along with that 3 new function have been added >>error() - returns an error code for the last error genterated >>error_string() - returns a string with a description of the error >>error_clear() - clears the last error >> >>Ryan > > > Not to be incredibly anal, but the .php file should not have any > newlines after the closing ?> tag -- this tends to mess with headers > should it be included after headers have been sent. :( > > -- Ryan Gilfether Software Engineer http://www.gilfether.com From brad@danga.com Mon Aug 11 06:50:44 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 22:50:44 -0700 (PDT) Subject: [test case] terrible performance with select() In-Reply-To: References: Message-ID: I couldn't reproduce this. But then, I can only get it to use poll, and not select. I haven't read libevent to see how it makes that decision. On Sun, 10 Aug 2003, Evan Martin wrote: > As a few people (including me) have found out, memcache is suspiciously > slow when using libevent backends other than epoll. (I think? I guess > I've only seen the behavior myself with select().) > > FWIW, it appears the problem that event_hander is taking up to five > seconds to get called, but only after repeat commands. Why that > happens is a more difficult problem, because it works fine with epoll > so it must be some behavior that's inconsistent underneath libevent. > > Attached is a test case that exhibits the behavior. Of note is the > perfectly round numbers that come up when it's being slow. Is > something timing out here? > (The "delay" here is the delay between subsequent requests.) > > lulu:~/projects/memcached/api/python% ./slow.py 1 > Trying get()s with 1.00s delay > get took 0.13 seconds > get took 0.30 seconds > get took 4.00 seconds > get took 4.00 seconds > get took 4.00 seconds > get took 4.00 seconds > lulu:~/projects/memcached/api/python% ./slow.py 0.9 > Trying get()s with 0.90s delay > get took 0.18 seconds > get took 1.38 seconds > get took 4.10 seconds > get took 4.10 seconds > get took 4.10 seconds > get took 4.10 seconds > > > From brad@danga.com Mon Aug 11 07:55:48 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Sun, 10 Aug 2003 23:55:48 -0700 (PDT) Subject: memcached-1.1.9-snapshot Message-ID: Here's a pre-release of 1.1.9. It's a pre-release because LiveJournal hasn't put it into production yet, and we might make a few things faster yet (packet coalescing), but with TCP_NODELAY on, it's much faster. Changes from 1.1.8: * TCP_NODELAY for better performance (avva) * support multiple levels of verbosity and make it not close stderr when in daemon mode if you're using verbose (evan, brad) * portability fixes to build on OS X / BSDs (evan) Download and enjoy: http://www.danga.com/memcached/dist/snapshots/memcached-snapshot.tar.gz The version number from the "stats" or "version" command is 1.1.9-snapshot, which means "not yet 1.1.9", not "1.1.9 and then some", which I believe is the proper naming convention? If not, I can change it in future snapshots. Please post experiences, good or bad. Thanks! - Brad From mellon@pobox.com Mon Aug 11 11:12:49 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Mon, 11 Aug 2003 10:12:49 +0000 Subject: Read vs. write performance (sloppiness in memcache) In-Reply-To: References: <20030811062755.GA7772@pobox.com> Message-ID: <20030811101249.GA8314@pobox.com> On Sun, Aug 10, 2003 at 08:56:17PM -0700, Brad Fitzpatrick wrote: > Yeah, this patch makes a world of difference. Instead of 20 seconds, it > now takes less than 1. > > However, it's even more packet-happy than before. > > Dude, we have a more-than-plentiful per-connect buffer on the server side > (DATA_BUFFER_SIZE = 2k, by default). Let's use as much of it as we can > before writing. Yes, you're right. That seems like a good idea, and it'll likely reduce the number of packets for large requests at no extra cost. Let me try to whip up a patch here... -- avva From brion@pobox.com Mon Aug 11 09:00:56 2003 From: brion@pobox.com (Brion Vibber) Date: Mon, 11 Aug 2003 01:00:56 -0700 Subject: memcached-1.1.9-snapshot In-Reply-To: References: Message-ID: <3F374D38.5090408@pobox.com> Brad Fitzpatrick wrote: >Changes from 1.1.8: > >* TCP_NODELAY for better performance (avva) > Yay! Runs great with my braindead fetch-500-tiny-separate-integers test. >* support multiple levels of verbosity and make it not > close stderr when in daemon mode if you're using verbose (evan, brad) > It's now very verbose. :) However it daemon mode it breaks: $ ./memcached -p 11000 -vvv -d <3 server listening memcached: kevent: Bad file descriptor Hmm, actually I can't get daemon mode to work without verbosity either on FreeBSD; it just seems to die silently, both with the snapshot version and 1.1.6. It works fine on Linux, though. >* portability fixes to build on OS X / BSDs (evan) > Doesn't quite compile on FreeBSD 5.1: gcc -DNDEBUG -g -O2 -I/usr/local/include -static -L/usr/local/lib -o memcached memcached.o slabs.o items.o assoc.o -levent memcached.o: In function `main': /home/brion/src/memc/memcached-1.1.9-snapshot/memcached.c:1299: undefined reference to `mlockall' *** Error code 1 If I comment out the mlockall() call, it compiles and works fine so far, though without the ability to lock the memory. This can probably be sensibly #ifdef'd out. (Also, I have to pass --with-libevent=/usr/local to configure, but that's normal enough.) -- brion vibber (brion @ pobox.com) From brion@pobox.com Mon Aug 11 09:24:55 2003 From: brion@pobox.com (Brion Vibber) Date: Mon, 11 Aug 2003 01:24:55 -0700 Subject: memcached-1.1.9-snapshot In-Reply-To: <3F374D38.5090408@pobox.com> References: <3F374D38.5090408@pobox.com> Message-ID: <3F3752D7.7010203@pobox.com> This is a multi-part message in MIME format. --------------000201060400080008040004 Content-Type: text/plain; charset=us-ascii; format=flowed Content-Transfer-Encoding: 7bit I wrote: > If I comment out the mlockall() call, it compiles and works fine so > far, though without the ability to lock the memory. This can probably > be sensibly #ifdef'd out. If there is such a thing as the opposite of an autoconf guru, that's me. Thanks to Google, I did figure out how to add a macro to the configure script to check for mlockall. However I'm not sure exactly what arcane series of commands is required to make it regenerate all the scripts and not give a bunch of errors about "./depcomp: ./depcomp: No such file or directory" I did manage to make a working binary on FreeBSD. On Linux, I got it to go through the configure script and agree that, yes, mlockall is supported, but the make dies looking for depcomp. Steps to reproduce: * apply attached patch * autoconf * automake * run one or more of the above commands over and over as it bitches at you that your versions are wrong and you should run aclocal then run it again * give up and hope configure works * ./configure (on FreeBSD, ./configure --with-event=/usr/local) * make * #^&*%$! * automake * make * if you're lucky, you might have a binary now. -- brion vibber (brion @ pobox.com) --------------000201060400080008040004 Content-Type: text/plain; name="mlockall-ac.diff" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="mlockall-ac.diff" --- plain/configure.ac Sun Aug 10 23:42:39 2003 +++ configure.ac Mon Aug 11 01:10:11 2003 @@ -56,5 +56,7 @@ AC_MSG_RESULT(yes) fi +AC_CHECK_FUNCS(mlockall) + AC_CONFIG_FILES(Makefile) AC_OUTPUT --- plain/memcached.c Sun Aug 10 23:45:08 2003 +++ memcached.c Mon Aug 11 01:10:30 2003 @@ -1296,7 +1296,9 @@ /* lock paged memory if needed */ if (lock_memory) { +#ifdef HAVE_MLOCKALL mlockall(MCL_CURRENT | MCL_FUTURE); +#endif } /* create the listening socket and bind it */ --------------000201060400080008040004-- From brad@danga.com Mon Aug 11 17:47:12 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 11 Aug 2003 09:47:12 -0700 (PDT) Subject: memcached-1.1.9-snapshot In-Reply-To: <3F374D38.5090408@pobox.com> References: <3F374D38.5090408@pobox.com> Message-ID: > >* TCP_NODELAY for better performance (avva) > > > Yay! Runs great with my braindead fetch-500-tiny-separate-integers test. It'll get faster once we combine packets. > >* support multiple levels of verbosity and make it not > > close stderr when in daemon mode if you're using verbose (evan, brad) > > > It's now very verbose. :) However it daemon mode it breaks: > > $ ./memcached -p 11000 -vvv -d > <3 server listening > memcached: kevent: Bad file descriptor > > Hmm, actually I can't get daemon mode to work without verbosity either > on FreeBSD; it just seems to die silently, both with the snapshot > version and 1.1.6. Hmm. You have a spare FreeBSD machine I can have a temporary account on? If so, I can mail you my ssh public key. > >* portability fixes to build on OS X / BSDs (evan) > > > Doesn't quite compile on FreeBSD 5.1: > > gcc -DNDEBUG -g -O2 -I/usr/local/include -static -L/usr/local/lib -o > memcached memcached.o slabs.o items.o assoc.o -levent > memcached.o: In function `main': > /home/brion/src/memc/memcached-1.1.9-snapshot/memcached.c:1299: > undefined reference to `mlockall' > *** Error code 1 > > If I comment out the mlockall() call, it compiles and works fine so far, > though without the ability to lock the memory. This can probably be > sensibly #ifdef'd out. I applied your patch, slightly modified. If you checkout from CVS, it'll work. I'll make a new snapshot later today. > (Also, I have to pass --with-libevent=/usr/local to configure, but > that's normal enough.) Is that normal? Anything we can do to avoid that? Well, I guess the FreeBSD ports tree will include it for you, huh? - Brad From memcached@abulman.co.uk Mon Aug 11 19:23:51 2003 From: memcached@abulman.co.uk (Alister) Date: Mon, 11 Aug 2003 19:23:51 +0100 Subject: PHP API 1.0.7 In-Reply-To: <3F371D94.5040100@gilfether.com> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> Message-ID: <20030811192351.36dfc541.alister@abulman.co.uk> On Mon, 11 Aug 2003 00:37:40 -0400 Ryan Gilfether wrote: > Lisa Marie Seelye wrote: > > Not to be incredibly anal, but the .php file should not have any > > newlines after the closing ?> tag -- this tends to mess with headers > > should it be included after headers have been sent. :( > yes, you're right, I didnt notice that a newline got included at the > end of the file. I'll be sure to remove it in future releases. Thanks > for pointing it out. I tend to just not put in the trailing ?> - if it's not there, you can't have errant spaces after it. Alister From brion@pobox.com Tue Aug 12 02:10:49 2003 From: brion@pobox.com (Brion Vibber) Date: Mon, 11 Aug 2003 18:10:49 -0700 Subject: memcached-1.1.9-snapshot In-Reply-To: References: <3F374D38.5090408@pobox.com> Message-ID: <3F383E99.2010505@pobox.com> Brad Fitzpatrick wrote: >I applied your patch, slightly modified. If you checkout from CVS, it'll >work. I'll make a new snapshot later today. > Waaah, there's no configure script! bash-2.05b$ autoconf configure.ac:4: error: possibly undefined macro: AM_INIT_AUTOMAKE configure.ac:5: error: possibly undefined macro: AM_CONFIG_HEADER bash-2.05b$ ./configure --with-libevent=/usr/local ./configure: line 1206: syntax error near unexpected token `memcached,' ./configure: line 1206: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' Hmm. Erase, check out again (can't do 'make distclean' cause there's no makefile!), start over. bash-2.05b$ autoconf configure.ac:4: error: possibly undefined macro: AM_INIT_AUTOMAKE configure.ac:5: error: possibly undefined macro: AM_CONFIG_HEADER bash-2.05b$ automake automake: configure.ac: required file `./install-sh' not found automake: configure.ac: required file `./mkinstalldirs' not found automake: configure.ac: required file `./missing' not found automake: Makefile.am: required file `./INSTALL' not found automake: Makefile.am: required file `./COPYING' not found configure.ac: 5: required file `./config.h.in' not found automake: configure.ac: required file `./depcomp' not found /usr/local/share/automake/am/depend2.am: AMDEP does not appear in AM_CONDITIONAL /usr/local/share/automake/am/depend2.am: AMDEP does not appear in AM_CONDITIONAL /usr/local/share/automake/am/depend2.am: AMDEP does not appear in AM_CONDITIONAL /usr/local/share/automake/am/lang-compile.am: AMDEP does not appear in AM_CONDITIONAL bash-2.05b$ ./configure ./configure: line 1206: syntax error near unexpected token `memcached,' ./configure: line 1206: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' bash-2.05b$ autoconf configure.ac:4: error: possibly undefined macro: AM_INIT_AUTOMAKE configure.ac:5: error: possibly undefined macro: AM_CONFIG_HEADER bash-2.05b$ aclocal bash-2.05b$ autoconf configure.ac:5: warning: do not use m4_patsubst: use patsubst or m4_bpatsubst configure.ac:62: warning: do not use m4_regexp: use regexp or m4_bregexp bash-2.05b$ ./configure configure: error: cannot find install-sh or install.sh in . ./.. ./../.. $ automake automake: configure.ac: required file `./install-sh' not found automake: configure.ac: required file `./mkinstalldirs' not found automake: configure.ac: required file `./missing' not found automake: Makefile.am: required file `./INSTALL' not found automake: Makefile.am: required file `./COPYING' not found configure.ac: 5: required file `./config.h.in' not found automake: configure.ac: required file `./depcomp' not found bash-2.05b$ ./configure configure: error: cannot find install-sh or install.sh in . ./.. ./../.. Give up, erase everything and check out again... bash-2.05b$ aclocal bash-2.05b$ autoheader configure.ac:5: warning: do not use m4_patsubst: use patsubst or m4_bpatsubst configure.ac:62: warning: do not use m4_regexp: use regexp or m4_bregexp autoheader: `config.h.in' is created bash-2.05b$ automake automake: configure.ac: required file `./install-sh' not found automake: configure.ac: required file `./mkinstalldirs' not found automake: configure.ac: required file `./missing' not found automake: Makefile.am: required file `./INSTALL' not found automake: Makefile.am: required file `./COPYING' not found automake: configure.ac: required file `./depcomp' not found bash-2.05b$ autoconf bash-2.05b$ ./configure --with-libevent=/usr/local configure: error: cannot find install-sh or install.sh in . ./.. ./../.. Did I mention I hate autoconf? :) -- brion vibber (brion @ pobox.com) From brad@danga.com Tue Aug 12 02:27:06 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 11 Aug 2003 18:27:06 -0700 (PDT) Subject: memcached-1.1.9-snapshot In-Reply-To: <3F383E99.2010505@pobox.com> References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> Message-ID: $ sh ./autogen.sh On Mon, 11 Aug 2003, Brion Vibber wrote: > Brad Fitzpatrick wrote: > > >I applied your patch, slightly modified. If you checkout from CVS, it'll > >work. I'll make a new snapshot later today. > > > > Waaah, there's no configure script! > > bash-2.05b$ autoconf > configure.ac:4: error: possibly undefined macro: AM_INIT_AUTOMAKE > configure.ac:5: error: possibly undefined macro: AM_CONFIG_HEADER > bash-2.05b$ ./configure --with-libevent=/usr/local > ./configure: line 1206: syntax error near unexpected token `memcached,' > ./configure: line 1206: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' > > Hmm. Erase, check out again (can't do 'make distclean' cause there's no > makefile!), start over. > > bash-2.05b$ autoconf > configure.ac:4: error: possibly undefined macro: AM_INIT_AUTOMAKE > configure.ac:5: error: possibly undefined macro: AM_CONFIG_HEADER > bash-2.05b$ automake > automake: configure.ac: required file `./install-sh' not found > automake: configure.ac: required file `./mkinstalldirs' not found > automake: configure.ac: required file `./missing' not found > automake: Makefile.am: required file `./INSTALL' not found > automake: Makefile.am: required file `./COPYING' not found > configure.ac: 5: required file `./config.h.in' not found > automake: configure.ac: required file `./depcomp' not found > /usr/local/share/automake/am/depend2.am: AMDEP does not appear in > AM_CONDITIONAL > /usr/local/share/automake/am/depend2.am: AMDEP does not appear in > AM_CONDITIONAL > /usr/local/share/automake/am/depend2.am: AMDEP does not appear in > AM_CONDITIONAL > /usr/local/share/automake/am/lang-compile.am: AMDEP does not appear in > AM_CONDITIONAL > bash-2.05b$ ./configure > ./configure: line 1206: syntax error near unexpected token `memcached,' > ./configure: line 1206: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' > bash-2.05b$ autoconf > configure.ac:4: error: possibly undefined macro: AM_INIT_AUTOMAKE > configure.ac:5: error: possibly undefined macro: AM_CONFIG_HEADER > bash-2.05b$ aclocal > bash-2.05b$ autoconf > configure.ac:5: warning: do not use m4_patsubst: use patsubst or > m4_bpatsubst > configure.ac:62: warning: do not use m4_regexp: use regexp or m4_bregexp > bash-2.05b$ ./configure > configure: error: cannot find install-sh or install.sh in . ./.. ./../.. > $ automake > automake: configure.ac: required file `./install-sh' not found > automake: configure.ac: required file `./mkinstalldirs' not found > automake: configure.ac: required file `./missing' not found > automake: Makefile.am: required file `./INSTALL' not found > automake: Makefile.am: required file `./COPYING' not found > configure.ac: 5: required file `./config.h.in' not found > automake: configure.ac: required file `./depcomp' not found > bash-2.05b$ ./configure > configure: error: cannot find install-sh or install.sh in . ./.. ./../.. > > Give up, erase everything and check out again... > > bash-2.05b$ aclocal > bash-2.05b$ autoheader > configure.ac:5: warning: do not use m4_patsubst: use patsubst or > m4_bpatsubst > configure.ac:62: warning: do not use m4_regexp: use regexp or m4_bregexp > autoheader: `config.h.in' is created > bash-2.05b$ automake > automake: configure.ac: required file `./install-sh' not found > automake: configure.ac: required file `./mkinstalldirs' not found > automake: configure.ac: required file `./missing' not found > automake: Makefile.am: required file `./INSTALL' not found > automake: Makefile.am: required file `./COPYING' not found > automake: configure.ac: required file `./depcomp' not found > bash-2.05b$ autoconf > bash-2.05b$ ./configure --with-libevent=/usr/local > configure: error: cannot find install-sh or install.sh in . ./.. ./../.. > > Did I mention I hate autoconf? :) > > -- brion vibber (brion @ pobox.com) > > From brion@pobox.com Tue Aug 12 02:40:11 2003 From: brion@pobox.com (Brion Vibber) Date: Mon, 11 Aug 2003 18:40:11 -0700 Subject: memcached-1.1.9-snapshot In-Reply-To: References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> Message-ID: <3F38457B.2090203@pobox.com> Brad Fitzpatrick wrote: >$ sh ./autogen.sh > $ sh ./autogen.sh aclocal... autoheader... autoheader: error: AC_CONFIG_HEADERS not found in configure.ac -- brion vibber (brion @ pobox.com) From martine@danga.com Tue Aug 12 02:46:53 2003 From: martine@danga.com (Evan Martin) Date: 11 Aug 2003 18:46:53 -0700 Subject: memcached-1.1.9-snapshot In-Reply-To: <3F38457B.2090203@pobox.com> References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> <3F38457B.2090203@pobox.com> Message-ID: <1060652813.23503.0.camel@trout> On Mon, 2003-08-11 at 18:40, Brion Vibber wrote: > Brad Fitzpatrick wrote: > > >$ sh ./autogen.sh > > > $ sh ./autogen.sh > aclocal... > autoheader... > autoheader: error: AC_CONFIG_HEADERS not found in configure.ac I was encountering this, too, but I thought it was my weird setup. Can you try running "aclocal" manually first and then running autogen again? -- Evan Martin martine@danga.com http://neugierig.org From brad@danga.com Tue Aug 12 02:51:07 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 11 Aug 2003 18:51:07 -0700 (PDT) Subject: memcached-1.1.9-snapshot In-Reply-To: <1060652813.23503.0.camel@trout> References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> <3F38457B.2090203@pobox.com> <1060652813.23503.0.camel@trout> Message-ID: I was running into "AC_CONFIG_HEADERS not found" a ton yesterday. I hate autoconf too. I got around it by running aclocal first, but I have no clue why that works. On Mon, 11 Aug 2003, Evan Martin wrote: > On Mon, 2003-08-11 at 18:40, Brion Vibber wrote: > > Brad Fitzpatrick wrote: > > > > >$ sh ./autogen.sh > > > > > $ sh ./autogen.sh > > aclocal... > > autoheader... > > autoheader: error: AC_CONFIG_HEADERS not found in configure.ac > > I was encountering this, too, but I thought it was my weird setup. > Can you try running "aclocal" manually first and then running autogen > again? > > -- > Evan Martin > martine@danga.com > http://neugierig.org > > From brion@pobox.com Tue Aug 12 03:10:35 2003 From: brion@pobox.com (Brion Vibber) Date: Mon, 11 Aug 2003 19:10:35 -0700 Subject: memcached-1.1.9-snapshot In-Reply-To: <1060652813.23503.0.camel@trout> References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> <3F38457B.2090203@pobox.com> <1060652813.23503.0.camel@trout> Message-ID: <3F384C9B.9030006@pobox.com> Evan Martin wrote: >On Mon, 2003-08-11 at 18:40, Brion Vibber wrote: > > >>$ sh ./autogen.sh >>aclocal... >>autoheader... >>autoheader: error: AC_CONFIG_HEADERS not found in configure.ac >> >> > >I was encountering this, too, but I thought it was my weird setup. >Can you try running "aclocal" manually first and then running autogen >again? > Yeah, oddly enough that works. Hmm... Let's look at that autogen.sh: #!/bin/sh # # This is hacky, because there are so many damn versions # of autoconf/automake. It works with Debian woody, at least. # echo "aclocal..." $ACLOCAL || aclocal-1.7 || aclocal-1.5 || aclocal || exit 1 Waitaminute... I ain't got no ACLOCAL env variable! If I run, say: $ $FOOBAR || echo hi || echo bob || echo foo I get nothin'. So, it's not running aclocal at all unless you set ACLOCAL: $ ACLOCAL=aclocal sh ./autogen.sh -- brion vibber (brion @ pobox.com) From brad@danga.com Tue Aug 12 03:22:36 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 11 Aug 2003 19:22:36 -0700 (PDT) Subject: memcached-1.1.9-snapshot In-Reply-To: <3F384C9B.9030006@pobox.com> References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> <3F38457B.2090203@pobox.com> <1060652813.23503.0.camel@trout> <3F384C9B.9030006@pobox.com> Message-ID: I'll fix that. It's still gonna suck, but maybe it'll suck less. On Mon, 11 Aug 2003, Brion Vibber wrote: > Evan Martin wrote: > > >On Mon, 2003-08-11 at 18:40, Brion Vibber wrote: > > > > > >>$ sh ./autogen.sh > >>aclocal... > >>autoheader... > >>autoheader: error: AC_CONFIG_HEADERS not found in configure.ac > >> > >> > > > >I was encountering this, too, but I thought it was my weird setup. > >Can you try running "aclocal" manually first and then running autogen > >again? > > > Yeah, oddly enough that works. > > Hmm... Let's look at that autogen.sh: > #!/bin/sh > # > # This is hacky, because there are so many damn versions > # of autoconf/automake. It works with Debian woody, at least. > # > > echo "aclocal..." > $ACLOCAL || aclocal-1.7 || aclocal-1.5 || aclocal || exit 1 > > Waitaminute... I ain't got no ACLOCAL env variable! If I run, say: > > $ $FOOBAR || echo hi || echo bob || echo foo > > I get nothin'. So, it's not running aclocal at all unless you set ACLOCAL: > > $ ACLOCAL=aclocal sh ./autogen.sh > > -- brion vibber (brion @ pobox.com) > > From mellon@pobox.com Sat Aug 16 03:30:27 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Sat, 16 Aug 2003 05:30:27 +0300 Subject: [patch] combining packets Message-ID: <20030816023027.GA17062@pobox.com> Here's an attempt to combine many smaller packets into one larger packet (in answers to GET queries... that's the only place where it matters). This patch also simplifies the conn_mwrite finite machine state considerably. Tested only cursorily so far, needs review and more testing. main -> wcmtools src/memcached/memcached.c --- cvs/wcmtools/memcached/memcached.c 2003-08-11 16:59:47.000000000 +0300 +++ src/memcached/memcached.c 2003-08-16 05:23:39.000000000 +0300 @@ -149,8 +149,6 @@ c->rcurr = c->rbuf; c->icurr = c->ilist; c->ileft = 0; - c->iptr = c->ibuf; - c->ibytes = 0; c->write_and_go = conn_read; c->write_and_free = 0; @@ -609,7 +607,8 @@ if (c->ileft) { c->ipart = 0; c->state = conn_mwrite; - c->ibytes = 0; + c->wbytes = 0; + c->wcurr = c->wbuf; return; } else { out_string(c, "END"); @@ -763,7 +762,7 @@ if (event_add(&c->event, 0) == -1) return 0; return 1; } - + void drive_machine(conn *c) { int exit = 0; @@ -919,6 +918,7 @@ c->write_and_free = 0; } c->state = c->write_and_go; + c->wcurr = c->wbuf; break; } res = write(c->sfd, c->wcurr, c->wbytes); @@ -946,74 +946,114 @@ break; case conn_mwrite: /* - * we're writing ibytes bytes from iptr. iptr alternates between - * ibuf, where we build a string "VALUE...", and ITEM_data(it) for the - * current item. When we finish a chunk, we choose the next one using + * we're writing wbytes bytes from wcurr. wcurr alternates between + * wbuf, in which we build a string "VALUE...", and item data for smaller items, + * and ITEM_data(it) for the larger items. When we finish a chunk, we choose the next one using * ipart, which has the following semantics: 0 - start the loop, 1 - * we finished ibuf, go to current ITEM_data(it); 2 - we finished ITEM_data(it), * move to the next item and build its ibuf; 3 - we finished all items, - * write "END". + * write "END". wcurr points inside wbuf when we accumulate + * data, but we restore it to =wbuf before actual writing. */ - if (c->ibytes > 0) { - res = write(c->sfd, c->iptr, c->ibytes); - if (res > 0) { - stats.bytes_written += res; - c->iptr += res; - c->ibytes -= res; - break; - } - if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { - if (!update_event(c, EV_WRITE | EV_PERSIST)) { - if (settings.verbose > 0) - fprintf(stderr, "Couldn't update event\n"); - c->state = conn_closing; - break; - } - exit = 1; - break; - } - /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, - we have a real error, on which we close the connection */ - if (settings.verbose > 0) - fprintf(stderr, "Failed to write, and not due to blocking\n"); - c->state = conn_closing; - break; - } else { + switch (c->ipart) { + int len, res; item *it; - /* we finished a chunk, decide what to do next */ - switch (c->ipart) { - case 1: - it = *(c->icurr); - assert((it->it_flags & ITEM_SLABBED) == 0); - c->iptr = ITEM_data(it); - c->ibytes = it->nbytes; + + case 1: + it = *(c->icurr); + assert((it->it_flags & ITEM_SLABBED) == 0); + + /* is there enough place in the fixed buffer for this item? */ + if (c->wsize - c->wbytes >= it->nbytes) { + memcpy(c->wcurr, ITEM_data(it), it->nbytes); + c->wbytes += it->nbytes; + c->wcurr += it->nbytes; c->ipart = 2; break; - case 2: - it = *(c->icurr); - item_remove(it); - c->ileft--; - if (c->ileft <= 0) { - c->ipart = 3; + } else { + if (c->wbytes) { + /* + * flush the buffer, then come back here and try the + * above size condition again. + */ + c->wcurr = c->wbuf; + c->state = conn_write; + c->write_and_go = conn_mwrite; break; } else { - c->icurr++; + /* + * the buffer is empty and the data is still + * larger than the buffer, so write it out + */ + c->wcurr = ITEM_data(it); + c->wbytes = it->nbytes; + c->state = conn_write; + c->write_and_go = conn_mwrite; + c->ipart = 2; + break; } - /* FALL THROUGH */ - case 0: - it = *(c->icurr); - assert((it->it_flags & ITEM_SLABBED) == 0); - sprintf(c->ibuf, "VALUE %s %u %u\r\n", ITEM_key(it), it->flags, it->nbytes - 2); + } + + case 2: + it = *(c->icurr); + item_remove(it); + c->ileft--; + if (c->ileft <= 0) { + c->ipart = 3; + break; + } else { + c->icurr++; + } + + /* + * break and come back to ipart=0, rather than fall + * through: in ipart=0, we may need to flush the write + * buffer and return to the same place, and if we didn't + * change ipart, we'd've returned to ipart=2 erroneously. + */ + + c->ipart = 0; + break; + + case 0: + it = *(c->icurr); + assert((it->it_flags & ITEM_SLABBED) == 0); + + len = strlen(ITEM_key(it)); + if (c->wsize - c->wbytes >= len+32) { + res = sprintf(c->wcurr, "VALUE %s %u %u\r\n", ITEM_key(it), it->flags, it->nbytes - 2); + c->wcurr += res; + c->wbytes += res; if (settings.verbose > 1) fprintf(stderr, ">%d sending key %s\n", c->sfd, ITEM_key(it)); - c->iptr = c->ibuf; - c->ibytes = strlen(c->iptr); c->ipart = 1; break; - case 3: - out_string(c, "END"); + } else { + if (c->wbytes) { + /* flush the buffer and come back here */ + c->wcurr = c->wbuf; + c->state = conn_write; + c->write_and_go = conn_mwrite; + break; + } else { + /* should never happen */ + c->state = conn_closing; + break; + } + } + + case 3: + /* flush the buffer and come back here */ + if (c->wbytes) { + c->wcurr = c->wbuf; + c->state = conn_write; + c->write_and_go = conn_mwrite; break; } + + /* write "END\r\n" and finish */ + out_string(c, "END"); + break; } break; main -> wcmtools src/memcached/memcached.h --- cvs/wcmtools/memcached/memcached.h 2003-07-22 12:14:33.000000000 +0300 +++ src/memcached/memcached.h 2003-08-16 05:23:54.000000000 +0300 @@ -113,9 +113,6 @@ item **icurr; int ileft; int ipart; /* 1 if we're writing a VALUE line, 2 if we're writing data */ - char ibuf[256]; /* for VALUE lines */ - char *iptr; - int ibytes; } conn; From mellon@pobox.com Sat Aug 16 02:57:56 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Sat, 16 Aug 2003 04:57:56 +0300 Subject: memcached-1.1.9-snapshot In-Reply-To: References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> <3F38457B.2090203@pobox.com> <1060652813.23503.0.camel@trout> Message-ID: <20030816015756.GA16838@pobox.com> On Mon, Aug 11, 2003 at 06:51:07PM -0700, Brad Fitzpatrick wrote: > I was running into "AC_CONFIG_HEADERS not found" a ton yesterday. I hate > autoconf too. There should be a T-shirt. -- autoconf haters of the world unite (ceci n'est pas un haiku) From brad@danga.com Tue Aug 12 03:58:16 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 11 Aug 2003 19:58:16 -0700 (PDT) Subject: [patch] combining packets In-Reply-To: <20030816023027.GA17062@pobox.com> References: <20030816023027.GA17062@pobox.com> Message-ID: It looks like the word "END\r\n" would still come in its own packet. Why don't you try to get that into there if there are 5 bytes free? Also, I don't see any work here about matching our output sizes to the MTU of the interface. Should we just go with TCP_CORK/TCP_NOPUSH instead? Any thoughts from people with experience? is TCP_CORK/etc widely supported enough? If this were ported to Windows, or commercial Unixes, do they have fun TCP options like that? On Sat, 16 Aug 2003, Anatoly Vorobey wrote: > Here's an attempt to combine many smaller packets into one larger packet > (in answers to GET queries... that's the only place where it matters). > This patch also simplifies the conn_mwrite finite machine state > considerably. Tested only cursorily so far, needs review and more > testing. > > main -> wcmtools src/memcached/memcached.c > --- cvs/wcmtools/memcached/memcached.c 2003-08-11 16:59:47.000000000 +0300 > +++ src/memcached/memcached.c 2003-08-16 05:23:39.000000000 +0300 > @@ -149,8 +149,6 @@ > c->rcurr = c->rbuf; > c->icurr = c->ilist; > c->ileft = 0; > - c->iptr = c->ibuf; > - c->ibytes = 0; > > c->write_and_go = conn_read; > c->write_and_free = 0; > @@ -609,7 +607,8 @@ > if (c->ileft) { > c->ipart = 0; > c->state = conn_mwrite; > - c->ibytes = 0; > + c->wbytes = 0; > + c->wcurr = c->wbuf; > return; > } else { > out_string(c, "END"); > @@ -763,7 +762,7 @@ > if (event_add(&c->event, 0) == -1) return 0; > return 1; > } > - > + > void drive_machine(conn *c) { > > int exit = 0; > @@ -919,6 +918,7 @@ > c->write_and_free = 0; > } > c->state = c->write_and_go; > + c->wcurr = c->wbuf; > break; > } > res = write(c->sfd, c->wcurr, c->wbytes); > @@ -946,74 +946,114 @@ > break; > case conn_mwrite: > /* > - * we're writing ibytes bytes from iptr. iptr alternates between > - * ibuf, where we build a string "VALUE...", and ITEM_data(it) for the > - * current item. When we finish a chunk, we choose the next one using > + * we're writing wbytes bytes from wcurr. wcurr alternates between > + * wbuf, in which we build a string "VALUE...", and item data for smaller items, > + * and ITEM_data(it) for the larger items. When we finish a chunk, we choose the next one using > * ipart, which has the following semantics: 0 - start the loop, 1 - > * we finished ibuf, go to current ITEM_data(it); 2 - we finished ITEM_data(it), > * move to the next item and build its ibuf; 3 - we finished all items, > - * write "END". > + * write "END". wcurr points inside wbuf when we accumulate > + * data, but we restore it to =wbuf before actual writing. > */ > - if (c->ibytes > 0) { > - res = write(c->sfd, c->iptr, c->ibytes); > - if (res > 0) { > - stats.bytes_written += res; > - c->iptr += res; > - c->ibytes -= res; > - break; > - } > - if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { > - if (!update_event(c, EV_WRITE | EV_PERSIST)) { > - if (settings.verbose > 0) > - fprintf(stderr, "Couldn't update event\n"); > - c->state = conn_closing; > - break; > - } > - exit = 1; > - break; > - } > - /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, > - we have a real error, on which we close the connection */ > - if (settings.verbose > 0) > - fprintf(stderr, "Failed to write, and not due to blocking\n"); > - c->state = conn_closing; > - break; > - } else { > + switch (c->ipart) { > + int len, res; > item *it; > - /* we finished a chunk, decide what to do next */ > - switch (c->ipart) { > - case 1: > - it = *(c->icurr); > - assert((it->it_flags & ITEM_SLABBED) == 0); > - c->iptr = ITEM_data(it); > - c->ibytes = it->nbytes; > + > + case 1: > + it = *(c->icurr); > + assert((it->it_flags & ITEM_SLABBED) == 0); > + > + /* is there enough place in the fixed buffer for this item? */ > + if (c->wsize - c->wbytes >= it->nbytes) { > + memcpy(c->wcurr, ITEM_data(it), it->nbytes); > + c->wbytes += it->nbytes; > + c->wcurr += it->nbytes; > c->ipart = 2; > break; > - case 2: > - it = *(c->icurr); > - item_remove(it); > - c->ileft--; > - if (c->ileft <= 0) { > - c->ipart = 3; > + } else { > + if (c->wbytes) { > + /* > + * flush the buffer, then come back here and try the > + * above size condition again. > + */ > + c->wcurr = c->wbuf; > + c->state = conn_write; > + c->write_and_go = conn_mwrite; > break; > } else { > - c->icurr++; > + /* > + * the buffer is empty and the data is still > + * larger than the buffer, so write it out > + */ > + c->wcurr = ITEM_data(it); > + c->wbytes = it->nbytes; > + c->state = conn_write; > + c->write_and_go = conn_mwrite; > + c->ipart = 2; > + break; > } > - /* FALL THROUGH */ > - case 0: > - it = *(c->icurr); > - assert((it->it_flags & ITEM_SLABBED) == 0); > - sprintf(c->ibuf, "VALUE %s %u %u\r\n", ITEM_key(it), it->flags, it->nbytes - 2); > + } > + > + case 2: > + it = *(c->icurr); > + item_remove(it); > + c->ileft--; > + if (c->ileft <= 0) { > + c->ipart = 3; > + break; > + } else { > + c->icurr++; > + } > + > + /* > + * break and come back to ipart=0, rather than fall > + * through: in ipart=0, we may need to flush the write > + * buffer and return to the same place, and if we didn't > + * change ipart, we'd've returned to ipart=2 erroneously. > + */ > + > + c->ipart = 0; > + break; > + > + case 0: > + it = *(c->icurr); > + assert((it->it_flags & ITEM_SLABBED) == 0); > + > + len = strlen(ITEM_key(it)); > + if (c->wsize - c->wbytes >= len+32) { > + res = sprintf(c->wcurr, "VALUE %s %u %u\r\n", ITEM_key(it), it->flags, it->nbytes - 2); > + c->wcurr += res; > + c->wbytes += res; > if (settings.verbose > 1) > fprintf(stderr, ">%d sending key %s\n", c->sfd, ITEM_key(it)); > - c->iptr = c->ibuf; > - c->ibytes = strlen(c->iptr); > c->ipart = 1; > break; > - case 3: > - out_string(c, "END"); > + } else { > + if (c->wbytes) { > + /* flush the buffer and come back here */ > + c->wcurr = c->wbuf; > + c->state = conn_write; > + c->write_and_go = conn_mwrite; > + break; > + } else { > + /* should never happen */ > + c->state = conn_closing; > + break; > + } > + } > + > + case 3: > + /* flush the buffer and come back here */ > + if (c->wbytes) { > + c->wcurr = c->wbuf; > + c->state = conn_write; > + c->write_and_go = conn_mwrite; > break; > } > + > + /* write "END\r\n" and finish */ > + out_string(c, "END"); > + break; > } > break; > > main -> wcmtools src/memcached/memcached.h > --- cvs/wcmtools/memcached/memcached.h 2003-07-22 12:14:33.000000000 +0300 > +++ src/memcached/memcached.h 2003-08-16 05:23:54.000000000 +0300 > @@ -113,9 +113,6 @@ > item **icurr; > int ileft; > int ipart; /* 1 if we're writing a VALUE line, 2 if we're writing data */ > - char ibuf[256]; /* for VALUE lines */ > - char *iptr; > - int ibytes; > > } conn; > > > From mellon@pobox.com Sat Aug 16 04:02:54 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Sat, 16 Aug 2003 06:02:54 +0300 Subject: [patch] combining packets In-Reply-To: References: <20030816023027.GA17062@pobox.com> Message-ID: <20030816030254.GA17367@pobox.com> You wrote on Mon, Aug 11, 2003 at 07:58:16PM -0700: > It looks like the word "END\r\n" would still come in its own packet. > Why don't you try to get that into there if there are 5 bytes free? Hmm, you're right. I'll fix that. > Also, I don't see any work here about matching our output sizes to the MTU > of the interface. Not yet. > Should we just go with TCP_CORK/TCP_NOPUSH instead? Beats me. But if we do, combining our writes seems to be useless (though the patch is still good, it cleans the code a lot, and it can be easily changed not to combine, or to decide at runtime). -- avva From brad@danga.com Tue Aug 12 04:05:11 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 11 Aug 2003 20:05:11 -0700 (PDT) Subject: [patch] combining packets In-Reply-To: <20030816030254.GA17367@pobox.com> References: <20030816023027.GA17062@pobox.com> <20030816030254.GA17367@pobox.com> Message-ID: > > Should we just go with TCP_CORK/TCP_NOPUSH instead? > > Beats me. But if we do, combining our writes seems to be useless (though > the patch is still good, it cleans the code a lot, and it can be easily > changed not to combine, or to decide at runtime). Definitely. Killing that ibuf/iptr feels nice. > > -- > avva > > From brion@pobox.com Tue Aug 12 11:06:22 2003 From: brion@pobox.com (Brion Vibber) Date: Tue, 12 Aug 2003 03:06:22 -0700 Subject: autoconf (was Re: memcached-1.1.9-snapshot) In-Reply-To: References: <3F374D38.5090408@pobox.com> <3F383E99.2010505@pobox.com> <3F38457B.2090203@pobox.com> <1060652813.23503.0.camel@trout> <3F384C9B.9030006@pobox.com> Message-ID: <3F38BC1E.5030301@pobox.com> Brad Fitzpatrick wrote: >I'll fix that. > >It's still gonna suck, but maybe it'll suck less. > Okay, now in CVS it's: echo "aclocal..." ACLOCAL=${ACLOCAL:-aclocal} $ACLOCAL || aclocal-1.7 || aclocal-1.5 || aclocal || exit 1 This runs, but if no ACLOCAL is set it'll shortcut the attempts at aclocal-1.7 and aclocal-1.5. If it makes sense to try them, it should perhaps be: ACLOCAL=${ACLOCAL:-aclocal-1.7} $ACLOCAL || aclocal-1.5 || aclocal || exit 1 ? Anatoly Vorobey wrote: >On Mon, Aug 11, 2003 at 06:51:07PM -0700, Brad Fitzpatrick wrote: > > >>I was running into "AC_CONFIG_HEADERS not found" a ton yesterday. I hate >>autoconf too. >> >> > >There should be a T-shirt. > > And now there is: http://www.cafeshops.com/leuksman.7112875 -- brion vibber (brion @ pobox.com) From brad@danga.com Tue Aug 12 20:58:36 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 12:58:36 -0700 (PDT) Subject: network performance measurements Message-ID: I benchmarked 3 different methods of doing network I/O: 1) the default way, with the Nagel algorithm. 2) using TCP_CORK (Linux, same as TCP_PUSH on BSD) 3) using TCP_NODELAY I measured both real time and number of packets on the wire. The test was doing 2,500 deletes, sets, and gets, then a 2,500 get_multi. Seconds Packets DEFAULT 102.48 22638 TCP_CORK 3.88 15105 TCP_CORK 3.87 15108 TCP_CORK 3.86 15105 TCP_NODELAY 3.99 20169 TCP_NODELAY 4.04 20170 TCP_NODELAY 4.00 20170 Clearly, TCP_CORK is the winner. I can't believe LiveJournal's running on the DEFAULT above. We'll be switching it to TCP_CORK today and will make a release once we find it stable. Patch is below. Avva, please review. --- memcached.c 11 Aug 2003 16:41:28 -0000 1.34 +++ memcached.c 12 Aug 2003 19:56:14 -0000 @@ -96,6 +96,15 @@ int freetotal; int freecurr; +void set_cork (conn *c, int val) { + if (val) val = 1; + if (c->is_corked == val) return; + c->is_corked = val; +#ifdef TCP_NOPUSH + setsockopt(c->sfd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val)); +#endif +} + void conn_init(void) { freetotal = 200; freecurr = 0; @@ -156,6 +165,9 @@ c->write_and_free = 0; c->item = 0; + c->write_and_uncork = 0; + c->is_corked = 0; + event_set(&c->event, sfd, event_flags, event_handler, (void *)c); c->ev_flags = event_flags; @@ -234,6 +246,7 @@ c->wbytes = len + 2; c->wcurr = c->wbuf; + c->write_and_uncork = 1; c->state = conn_write; c->write_and_go = conn_read; return; @@ -475,6 +488,10 @@ if (settings.verbose > 1) fprintf(stderr, "<%d %s\n", c->sfd, command); + /* All incoming commands will require a response, so we cork at the beginning, + and uncork at the very end (usually by means of out_string) */ + set_cork(c, 1); + if ((strncmp(command, "add ", 4) == 0 && (comm = NREAD_ADD)) || (strncmp(command, "set ", 4) == 0 && (comm = NREAD_SET)) || (strncmp(command, "replace ", 8) == 0 && (comm = NREAD_REPLACE))) { @@ -926,6 +943,8 @@ stats.bytes_written += res; c->wcurr += res; c->wbytes -= res; + if (c->wbytes == 0 && c->write_and_uncork) + set_cork(c, 0); break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { Index: memcached.h =================================================================== RCS file: /home/cvspub/wcmtools/memcached/memcached.h,v retrieving revision 1.17 diff -u -r1.17 memcached.h --- memcached.h 12 Aug 2003 18:10:55 -0000 1.17 +++ memcached.h 12 Aug 2003 19:56:14 -0000 @@ -3,6 +3,10 @@ #define DATA_BUFFER_SIZE 2048 +#if defined(TCP_CORK) && !defined(TCP_NOPUSH) +#define TCP_NOPUSH TCP_CORK +#endif + struct stats { unsigned int curr_items; unsigned int total_items; @@ -89,6 +93,8 @@ int wbytes; int write_and_go; /* which state to go into after finishing current write */ void *write_and_free; /* free this memory after finishing writing */ + char write_and_uncork; /* boolean, uncork when done writing */ + char is_corked; /* boolean, connection is corked */ char *rcurr; int rlbytes; From jmat@shutdown.net Tue Aug 12 21:32:40 2003 From: jmat@shutdown.net (Justin Matlock) Date: Tue, 12 Aug 2003 16:32:40 -0400 Subject: possible bug in PHP API 1.0.8 References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> Message-ID: <002701c36110$e0584c80$0b00010a@nomium.net> Sorry for the extra-long email -- I just wanted to explain my thinking here, and go into excessive detail so someone can prove me wrong. :) I'm using the latest CVS snapshot of memcached, and the 1.0.8 PHP API, and the latest PHP 4.3.3 CVS tree, under Linux 2.4.21, Apache 2.0.47. It was all working great on my devel box, but when I moved it over to production, I started getting a ton of "_load_items(): Bytes read is greater than requested data size." errors. But the thing is: the development box and production box are using the exact same memcached server. They run the same OS, the same PHP version, the same *everything*. They're even the same hardware (one serial number apart). The *only* difference is the production box is on the same subnet as the memcached server -- the development one is on a different one. Of course, I start doing a lot more testing on the development box, and I notice I was getting them over there too, just not as many. So I modified the script to give me more information (patch follows): ================= 1063,1064c1063,1064 < $this->_debug("_load_items(): Bytes read is greater than requested data size."); < --- > $this->_debug("_load_items(): Bytes read ($bytes_read) is greater than requested data size ($len)."); > $this->_debug("_load_items(): I received \"".urlencode($buf)."\""); ================= Here's what I'm getting in my logs (I modified the debug handler to use my custom logging routines) -- add 7 for the "\r\nEND\r\n". So we're getting back 2 characters instead of 1: (Data is "0") [03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes read (9) is greater than requested data size (1). [03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I received "%0A2%0D%0AEND%0D%0A" (Data is "-1") [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes read (10) is greater than requested data size (2). [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I received "%0A-1%0D%0AEND%0D%0A" (Data is a whole lot more than "0" :) ): [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes read (165) is greater than requested data size (157). [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I received "%0Aa%3A1%3A%7Bi%3A0%3Ba%3A5%3A%7Bs%3A7%3A%(for clarity, I cut off the end) The thing that sucks is I can't replicate this reliably. I can hit reload over and over, and it will only happen 1 out of every 10 or so times, on random data coming back. It's not just numerics, not text... I sniffed the connection, and here's one that barfed coming back from the server: VALUE profile_1_7028277b0e9405414ea71de0bdde789f_rank 0 2\r\n-1\r\nEND\r\n Looks okay to me; it follows the protocol, and there are two characters in the response. Memcached isn't the culprit. So on to the API... The problem appears to be with this line: $line = substr($line, strpos($line, "\r\n")+1, strlen($line)); I added some more logging lines before and after it to see what the 'pre' and 'post' parsing results were. By only doing +1, you're just stripping off the \r. It's always letting a \n through, even though it should be stripping it off. The strange thing to me is sometimes it only lets a \n through, but sometimes it lets "\n0\r\nEND\r\n" through (this is when it breaks). I'm still trying to track down why this is happening... In summary; I changed that +1 to a +2, and it always strips off the remaining \n. From there, it works great. I just pushed about 300 pages, and all came back without errors... Final patch: ============== 999c999 < $line = substr($line, strpos($line, "\r\n")+1, strlen($line)); --- > $line = substr($line, strpos($line, "\r\n")+2, strlen($line)); =============== "It works for me". :) Justin ----- Original Message ----- From: "Ryan Gilfether" To: Sent: Monday, August 11, 2003 12:37 AM Subject: Re: PHP API 1.0.7 > yes, you're right, I didnt notice that a newline got included at the end > of the file. I'll be sure to remove it in future releases. Thanks for > pointing it out. > > Ryan > > Lisa Marie Seelye wrote: > > On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: > > > >>A new version of the PHP API has been completed, verson 1.0.7. > >>I've had some requests for adding PHPDocs style comments to the API. > >>Along with that 3 new function have been added > >>error() - returns an error code for the last error genterated > >>error_string() - returns a string with a description of the error > >>error_clear() - clears the last error > >> > >>Ryan > > > > > > Not to be incredibly anal, but the .php file should not have any > > newlines after the closing ?> tag -- this tends to mess with headers > > should it be included after headers have been sent. :( > > > > > > -- > Ryan Gilfether > Software Engineer > http://www.gilfether.com > > From brad@danga.com Tue Aug 12 21:43:09 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 13:43:09 -0700 (PDT) Subject: possible bug in PHP API 1.0.8 In-Reply-To: <002701c36110$e0584c80$0b00010a@nomium.net> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> <002701c36110$e0584c80$0b00010a@nomium.net> Message-ID: Justin, Thanks! The PHP client is known to be buggy (discovered last week), and Ryan said he's working on it. Perhaps this is the only fix needed.... Ryan? - Brad On Tue, 12 Aug 2003, Justin Matlock wrote: > Sorry for the extra-long email -- I just wanted to explain my thinking here, > and go into excessive detail so someone can prove me wrong. :) > > I'm using the latest CVS snapshot of memcached, and the 1.0.8 PHP API, and > the latest PHP 4.3.3 CVS tree, under Linux 2.4.21, Apache 2.0.47. It was > all working great on my devel box, but when I moved it over to production, I > started getting a ton of "_load_items(): Bytes read is greater than > requested data size." errors. But the thing is: the development box and > production box are using the exact same memcached server. They run the same > OS, the same PHP version, the same *everything*. They're even the same > hardware (one serial number apart). The *only* difference is the production > box is on the same subnet as the memcached server -- the development one is > on a different one. Of course, I start doing a lot more testing on the > development box, and I notice I was getting them over there too, just not as > many. > > So I modified the script to give me more information (patch follows): > > ================= > 1063,1064c1063,1064 > < $this->_debug("_load_items(): Bytes > read is greater than requested data size."); > < > --- > > $this->_debug("_load_items(): Bytes > read ($bytes_read) is greater than requested data size ($len)."); > > $this->_debug("_load_items(): I > received \"".urlencode($buf)."\""); > ================= > > Here's what I'm getting in my logs (I modified the debug handler to use my > custom logging routines) -- add 7 for the "\r\nEND\r\n". So we're getting > back 2 characters instead of 1: > > (Data is "0") > [03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes > read (9) is greater than requested data size (1). > [03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I > received "%0A2%0D%0AEND%0D%0A" > > (Data is "-1") > [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes > read (10) is greater than requested data size (2). > [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I > received "%0A-1%0D%0AEND%0D%0A" > > (Data is a whole lot more than "0" :) ): > [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes > read (165) is greater than requested data size (157). > [03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I > received "%0Aa%3A1%3A%7Bi%3A0%3Ba%3A5%3A%7Bs%3A7%3A%(for clarity, I cut off > the end) > > The thing that sucks is I can't replicate this reliably. I can hit reload > over and over, and it will only happen 1 out of every 10 or so times, on > random data coming back. It's not just numerics, not text... > > I sniffed the connection, and here's one that barfed coming back from the > server: > > VALUE profile_1_7028277b0e9405414ea71de0bdde789f_rank 0 2\r\n-1\r\nEND\r\n > > Looks okay to me; it follows the protocol, and there are two characters in > the response. Memcached isn't the culprit. So on to the API... > > The problem appears to be with this line: > $line = substr($line, strpos($line, "\r\n")+1, strlen($line)); > > I added some more logging lines before and after it to see what the 'pre' > and 'post' parsing results were. > > By only doing +1, you're just stripping off the \r. It's always letting a > \n through, even though it should be stripping it off. The strange thing to > me is sometimes it only lets a \n through, but sometimes it lets > "\n0\r\nEND\r\n" through (this is when it breaks). I'm still trying to > track down why this is happening... > > In summary; I changed that +1 to a +2, and it always strips off the > remaining \n. From there, it works great. I just pushed about 300 pages, > and all came back without errors... > > Final patch: > ============== > 999c999 > < $line = substr($line, strpos($line, > "\r\n")+1, strlen($line)); > --- > > $line = substr($line, strpos($line, > "\r\n")+2, strlen($line)); > =============== > > "It works for me". :) > > Justin > > ----- Original Message ----- > From: "Ryan Gilfether" > To: > Sent: Monday, August 11, 2003 12:37 AM > Subject: Re: PHP API 1.0.7 > > > > yes, you're right, I didnt notice that a newline got included at the end > > of the file. I'll be sure to remove it in future releases. Thanks for > > pointing it out. > > > > Ryan > > > > Lisa Marie Seelye wrote: > > > On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: > > > > > >>A new version of the PHP API has been completed, verson 1.0.7. > > >>I've had some requests for adding PHPDocs style comments to the API. > > >>Along with that 3 new function have been added > > >>error() - returns an error code for the last error genterated > > >>error_string() - returns a string with a description of the error > > >>error_clear() - clears the last error > > >> > > >>Ryan > > > > > > > > > Not to be incredibly anal, but the .php file should not have any > > > newlines after the closing ?> tag -- this tends to mess with headers > > > should it be included after headers have been sent. :( > > > > > > > > > > -- > > Ryan Gilfether > > Software Engineer > > http://www.gilfether.com > > > > > > From lisa@gentoo.org Tue Aug 12 21:53:23 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Tue, 12 Aug 2003 16:53:23 -0400 Subject: possible bug in PHP API 1.0.8 In-Reply-To: <002701c36110$e0584c80$0b00010a@nomium.net> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> <002701c36110$e0584c80$0b00010a@nomium.net> Message-ID: <1060721603.1821.7.camel@lisa.thedoh.com> --=-4dNSUDm+IL1RzUWpk0B8 Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Tue, 2003-08-12 at 16:32, Justin Matlock wrote: > Sorry for the extra-long email -- I just wanted to explain my thinking he= re, > and go into excessive detail so someone can prove me wrong. :) >=20 > I'm using the latest CVS snapshot of memcached, and the 1.0.8 PHP API, an= d > the latest PHP 4.3.3 CVS tree, under Linux 2.4.21, Apache 2.0.47. It was > all working great on my devel box, but when I moved it over to production= , I > started getting a ton of "_load_items(): Bytes read is greater than > requested data size." errors. But the thing is: the development box and > production box are using the exact same memcached server. They run the s= ame > OS, the same PHP version, the same *everything*. They're even the same > hardware (one serial number apart). The *only* difference is the product= ion > box is on the same subnet as the memcached server -- the development one = is > on a different one. Of course, I start doing a lot more testing on the > development box, and I notice I was getting them over there too, just not= as > many. I got this error in the 1.0.7 version of the PHP API as well. Memcached is listening on 127.0.0.1:11211, using mod_php 4.3.2 and Apache 2.0.47. I was unable to pinpoint the problem. --=20 Regards, -Lisa --=-4dNSUDm+IL1RzUWpk0B8 Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/OVPDy0a1Vh5Jb8URArx4AJ0aeCEY698YsYwmQE3gIktnG2EYgQCfeRgc eXNtKp5jSFsDihOyP1eXE+0= =m8n4 -----END PGP SIGNATURE----- --=-4dNSUDm+IL1RzUWpk0B8-- From lisa@gentoo.org Tue Aug 12 21:57:30 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Tue, 12 Aug 2003 16:57:30 -0400 Subject: [patch] combining packets In-Reply-To: <20030816023027.GA17062@pobox.com> References: <20030816023027.GA17062@pobox.com> Message-ID: <1060721850.1819.12.camel@lisa.thedoh.com> --=-qmAasCiKQJneWSihhp1N Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Fri, 2003-08-15 at 22:30, Anatoly Vorobey wrote: > Here's an attempt to combine many smaller packets into one larger packet=20 > (in answers to GET queries... that's the only place where it matters). > This patch also simplifies the conn_mwrite finite machine state=20 > considerably. Tested only cursorily so far, needs review and more=20 > testing. There's also the possibility to further use the ASCII character set to "compress" the data stream. Using special characters to separate fields or to terminate fields, such as the null character (\0), `end of transmission' (0x04) instead of \r\n. One byte will definitely cut back on the amount of space in each packet (albeit, by a tiny amount). =20 --=20 Regards, -Lisa --=-qmAasCiKQJneWSihhp1N Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/OVS6y0a1Vh5Jb8URAu8fAJ9hGECGWEX0O7QKqpnGbDEV4mdsCQCgtvV3 HDXtBG73jvESh+2qCYI0zyc= =9pit -----END PGP SIGNATURE----- --=-qmAasCiKQJneWSihhp1N-- From vibber@aludra.usc.edu Tue Aug 12 23:46:29 2003 From: vibber@aludra.usc.edu (Brion Vibber) Date: Tue, 12 Aug 2003 15:46:29 -0700 (PDT) Subject: memcached on MacOS X Message-ID: For an unrelated project I'm considering using memcached on a MacOS X box to spare me from the slowness that is FileMaker Pro. With a just-checked-out CVS I can configure and compile, but it fails to link if set to produce a static executable, which is the default: gcc -DNDEBUG -g -O2 -static -o memcached memcached.o slabs.o items.o assoc.o -levent ld: can't locate file for: -lcrt0.o make[1]: *** [memcached] Error 1 make: *** [all] Error 2 I'm running OS X 10.2.6; I've got Fink installed, but it shouldn't interfere with anything. If I pass --disable-static to configure, memcached compiles, links, and runs. I haven't tested it in detail yet, but unlike FreeBSD the daemon mode does work. -- brion vibber (brion @ pobox.com) From brad@danga.com Tue Aug 12 23:57:36 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 15:57:36 -0700 (PDT) Subject: memcached on MacOS X In-Reply-To: References: Message-ID: FWIW, Evan ran into the same problem. On Tue, 12 Aug 2003, Brion Vibber wrote: > For an unrelated project I'm considering using memcached on a MacOS X box > to spare me from the slowness that is FileMaker Pro. > > With a just-checked-out CVS I can configure and compile, but it fails to > link if set to produce a static executable, which is the default: > > gcc -DNDEBUG -g -O2 -static -o memcached memcached.o slabs.o items.o > assoc.o -levent > ld: can't locate file for: -lcrt0.o > make[1]: *** [memcached] Error 1 > make: *** [all] Error 2 > > I'm running OS X 10.2.6; I've got Fink installed, but it shouldn't > interfere with anything. > > If I pass --disable-static to configure, memcached compiles, links, and > runs. I haven't tested it in detail yet, but unlike FreeBSD the daemon > mode does work. > > -- brion vibber (brion @ pobox.com) > > From ryan@gilfether.com Wed Aug 13 00:04:22 2003 From: ryan@gilfether.com (Ryan Gilfether) Date: Tue, 12 Aug 2003 19:04:22 -0400 Subject: possible bug in PHP API 1.0.8 Message-ID: <3F397276.5010204@gilfether.com> Hey, sorry for the late response, I got off work late. I'm looking into it right now. Ryan -- Ryan Gilfether Software Engineer http://www.gilfether.com From ryan@gilfether.com Wed Aug 13 00:25:17 2003 From: ryan@gilfether.com (Ryan Gilfether) Date: Tue, 12 Aug 2003 19:25:17 -0400 Subject: possible bug in PHP API 1.0.8 In-Reply-To: References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> <002701c36110$e0584c80$0b00010a@nomium.net> Message-ID: <3F39775D.6070309@gilfether.com> This is a multi-part message in MIME format. --------------020001090705080800040401 Content-Type: text/plain; charset=us-ascii; format=flowed Content-Transfer-Encoding: 7bit Attached is a patch that fixes the problem found by Justin. His solution fixed the problem. Thanks, Ryan Brad Fitzpatrick wrote: > Justin, > > Thanks! The PHP client is known to be buggy (discovered last week), and > Ryan said he's working on it. Perhaps this is the only fix needed.... > > Ryan? > > - Brad > > > On Tue, 12 Aug 2003, Justin Matlock wrote: > > >>Sorry for the extra-long email -- I just wanted to explain my thinking here, >>and go into excessive detail so someone can prove me wrong. :) >> >>I'm using the latest CVS snapshot of memcached, and the 1.0.8 PHP API, and >>the latest PHP 4.3.3 CVS tree, under Linux 2.4.21, Apache 2.0.47. It was >>all working great on my devel box, but when I moved it over to production, I >>started getting a ton of "_load_items(): Bytes read is greater than >>requested data size." errors. But the thing is: the development box and >>production box are using the exact same memcached server. They run the same >>OS, the same PHP version, the same *everything*. They're even the same >>hardware (one serial number apart). The *only* difference is the production >>box is on the same subnet as the memcached server -- the development one is >>on a different one. Of course, I start doing a lot more testing on the >>development box, and I notice I was getting them over there too, just not as >>many. >> >>So I modified the script to give me more information (patch follows): >> >>================= >>1063,1064c1063,1064 >>< $this->_debug("_load_items(): Bytes >>read is greater than requested data size."); >>< >>--- >> >>> $this->_debug("_load_items(): Bytes >> >>read ($bytes_read) is greater than requested data size ($len)."); >> >>> $this->_debug("_load_items(): I >> >>received \"".urlencode($buf)."\""); >>================= >> >>Here's what I'm getting in my logs (I modified the debug handler to use my >>custom logging routines) -- add 7 for the "\r\nEND\r\n". So we're getting >>back 2 characters instead of 1: >> >>(Data is "0") >>[03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes >>read (9) is greater than requested data size (1). >>[03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I >>received "%0A2%0D%0AEND%0D%0A" >> >>(Data is "-1") >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes >>read (10) is greater than requested data size (2). >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I >>received "%0A-1%0D%0AEND%0D%0A" >> >>(Data is a whole lot more than "0" :) ): >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes >>read (165) is greater than requested data size (157). >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I >>received "%0Aa%3A1%3A%7Bi%3A0%3Ba%3A5%3A%7Bs%3A7%3A%(for clarity, I cut off >>the end) >> >>The thing that sucks is I can't replicate this reliably. I can hit reload >>over and over, and it will only happen 1 out of every 10 or so times, on >>random data coming back. It's not just numerics, not text... >> >>I sniffed the connection, and here's one that barfed coming back from the >>server: >> >>VALUE profile_1_7028277b0e9405414ea71de0bdde789f_rank 0 2\r\n-1\r\nEND\r\n >> >>Looks okay to me; it follows the protocol, and there are two characters in >>the response. Memcached isn't the culprit. So on to the API... >> >>The problem appears to be with this line: >>$line = substr($line, strpos($line, "\r\n")+1, strlen($line)); >> >>I added some more logging lines before and after it to see what the 'pre' >>and 'post' parsing results were. >> >>By only doing +1, you're just stripping off the \r. It's always letting a >>\n through, even though it should be stripping it off. The strange thing to >>me is sometimes it only lets a \n through, but sometimes it lets >>"\n0\r\nEND\r\n" through (this is when it breaks). I'm still trying to >>track down why this is happening... >> >>In summary; I changed that +1 to a +2, and it always strips off the >>remaining \n. From there, it works great. I just pushed about 300 pages, >>and all came back without errors... >> >>Final patch: >>============== >>999c999 >>< $line = substr($line, strpos($line, >>"\r\n")+1, strlen($line)); >>--- >> >>> $line = substr($line, strpos($line, >> >>"\r\n")+2, strlen($line)); >>=============== >> >>"It works for me". :) >> >>Justin >> >>----- Original Message ----- >>From: "Ryan Gilfether" >>To: >>Sent: Monday, August 11, 2003 12:37 AM >>Subject: Re: PHP API 1.0.7 >> >> >> >>>yes, you're right, I didnt notice that a newline got included at the end >>>of the file. I'll be sure to remove it in future releases. Thanks for >>>pointing it out. >>> >>>Ryan >>> >>>Lisa Marie Seelye wrote: >>> >>>>On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: >>>> >>>> >>>>>A new version of the PHP API has been completed, verson 1.0.7. >>>>>I've had some requests for adding PHPDocs style comments to the API. >>>>>Along with that 3 new function have been added >>>>>error() - returns an error code for the last error genterated >>>>>error_string() - returns a string with a description of the error >>>>>error_clear() - clears the last error >>>>> >>>>>Ryan >>>> >>>> >>>>Not to be incredibly anal, but the .php file should not have any >>>>newlines after the closing ?> tag -- this tends to mess with headers >>>>should it be included after headers have been sent. :( >>>> >>>> >>> >>>-- >>>Ryan Gilfether >>>Software Engineer >>>http://www.gilfether.com >>> >>> >> >> > > -- Ryan Gilfether Software Engineer http://www.gilfether.com --------------020001090705080800040401 Content-Type: text/plain; name="php-memcached-1.0.8.diff" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="php-memcached-1.0.8.diff" --- devel/html/memcached/MemCachedClient.inc.php 2003-08-12 19:17:06.000000000 -0400 +++ MemCachedClient.inc.php 2003-08-11 00:27:44.000000000 -0400 @@ -602,7 +602,7 @@ } // connect to the server, if it fails, add it to the host_dead below - $sock = socket_create (AF_INET, SOCK_STREAM, SOL_TCP); + $sock = socket_create (AF_INET, SOCK_STREAM, getprotobyname("TCP")); // we need surpress the error message if a connection fails if(!@socket_connect($sock, $conn[0], $conn[1])) @@ -996,7 +996,7 @@ $bytes_read = 0; // get the left over data after the header is read - $line = substr($line, strpos($line, "\r\n")+2, strlen($line)); + $line = substr($line, strpos($line, "\r\n")+1, strlen($line)); } else { --------------020001090705080800040401-- From mellon@pobox.com Wed Aug 13 00:23:50 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Wed, 13 Aug 2003 02:23:50 +0300 Subject: network performance measurements In-Reply-To: References: Message-ID: <20030812232350.GA24758@pobox.com> On Tue, Aug 12, 2003 at 12:58:36PM -0700, Brad Fitzpatrick wrote: > Patch is below. Avva, please review. You only uncork in out_string(). Not all response paths end in out_string(), though almost all do. One exception I found is "stats slabs", there may be others. Besides this problem, the patch seems fine. -- avva From martine@danga.com Wed Aug 13 00:37:48 2003 From: martine@danga.com (Evan Martin) Date: 12 Aug 2003 16:37:48 -0700 Subject: memcached on MacOS X In-Reply-To: References: Message-ID: <1060731467.4069.10.camel@trout> On Tue, 2003-08-12 at 15:46, Brion Vibber wrote: > For an unrelated project I'm considering using memcached on a MacOS X box > to spare me from the slowness that is FileMaker Pro. > > With a just-checked-out CVS I can configure and compile, but it fails to > link if set to produce a static executable, which is the default: > > gcc -DNDEBUG -g -O2 -static -o memcached memcached.o slabs.o items.o > assoc.o -levent > ld: can't locate file for: -lcrt0.o > make[1]: *** [memcached] Error 1 > make: *** [all] Error 2 This is exactly the problem I had, and I couldn't figure it out. If you have any hints, I'm all ears--- Ooh, wait, I found it! http://developer.apple.com/qa/qa2001/qa1118.html Q: I'm trying to link my binary statically, but it's failing to link because it can't find 'crt0.o.' Why? A: Static linking of user binaries is not supported on Mac OS X. Tying user binaries to the internal implementation of Mac OS X libraries and interfaces would limit our ability to update and enhance Mac OS X. Instead, dynamic linking is supported (linking against crt1.o automatically instead of looking for crt0.o, for example). Hah! That's funny. I wonder if it's a Darwin limitation, or...? On the TODO for me, then: have autoconf detect you're on Darwin and force the static flag off. -- Evan Martin martine@danga.com http://neugierig.org From brad@danga.com Wed Aug 13 00:42:32 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 16:42:32 -0700 (PDT) Subject: memcached on MacOS X In-Reply-To: <1060731467.4069.10.camel@trout> References: <1060731467.4069.10.camel@trout> Message-ID: Go autoconf man! On Tue, 12 Aug 2003, Evan Martin wrote: > On Tue, 2003-08-12 at 15:46, Brion Vibber wrote: > > For an unrelated project I'm considering using memcached on a MacOS X box > > to spare me from the slowness that is FileMaker Pro. > > > > With a just-checked-out CVS I can configure and compile, but it fails to > > link if set to produce a static executable, which is the default: > > > > gcc -DNDEBUG -g -O2 -static -o memcached memcached.o slabs.o items.o > > assoc.o -levent > > ld: can't locate file for: -lcrt0.o > > make[1]: *** [memcached] Error 1 > > make: *** [all] Error 2 > > This is exactly the problem I had, and I couldn't figure it out. > If you have any hints, I'm all ears--- > > Ooh, wait, I found it! > http://developer.apple.com/qa/qa2001/qa1118.html > > Q: I'm trying to link my binary statically, but it's failing to link > because it can't find 'crt0.o.' Why? > > A: Static linking of user binaries is not supported on Mac OS X. Tying > user binaries to the internal implementation of Mac OS X libraries and > interfaces would limit our ability to update and enhance Mac OS X. > Instead, dynamic linking is supported (linking against crt1.o > automatically instead of looking for crt0.o, for example). > > Hah! That's funny. I wonder if it's a Darwin limitation, or...? > > On the TODO for me, then: have autoconf detect you're on Darwin and > force the static flag off. > > -- > Evan Martin > martine@danga.com > http://neugierig.org > > From brad@danga.com Wed Aug 13 00:43:49 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 16:43:49 -0700 (PDT) Subject: network performance measurements In-Reply-To: <20030812232350.GA24758@pobox.com> References: <20030812232350.GA24758@pobox.com> Message-ID: I was thinking: how about I just ditch c->write_and_uncork and check instead of c->write_and_go == conn_read? If the state machine is moving from writing to reading, it's done writing. I'll do that. Reply if you can think of a case where that's the wrong thing, let me know. On Wed, 13 Aug 2003, Anatoly Vorobey wrote: > On Tue, Aug 12, 2003 at 12:58:36PM -0700, Brad Fitzpatrick wrote: > > Patch is below. Avva, please review. > > You only uncork in out_string(). Not all response paths end in > out_string(), though almost all do. One exception I found is > "stats slabs", there may be others. > > Besides this problem, the patch seems fine. > > -- > avva > > From brad@danga.com Wed Aug 13 00:53:07 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 16:53:07 -0700 (PDT) Subject: possible bug in PHP API 1.0.8 In-Reply-To: <3F39775D.6070309@gilfether.com> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> <002701c36110$e0584c80$0b00010a@nomium.net> <3F39775D.6070309@gilfether.com> Message-ID: Your patches are consistently backwards. Not sure how you're making them, but try doing it in reverse. In any case, applied. In the future, update ChangeLog. And update MC_VERSION if you want me to make a new release, which I assume you do with this bug. Not sure why you changed getprotobyname("TCP") to SOL_TCP, but I included it also. New release will be on the site shortly. On Tue, 12 Aug 2003, Ryan Gilfether wrote: > Attached is a patch that fixes the problem found by Justin. His solution > fixed the problem. > > Thanks, > Ryan > > Brad Fitzpatrick wrote: > > Justin, > > > > Thanks! The PHP client is known to be buggy (discovered last week), and > > Ryan said he's working on it. Perhaps this is the only fix needed.... > > > > Ryan? > > > > - Brad > > > > > > On Tue, 12 Aug 2003, Justin Matlock wrote: > > > > > >>Sorry for the extra-long email -- I just wanted to explain my thinking here, > >>and go into excessive detail so someone can prove me wrong. :) > >> > >>I'm using the latest CVS snapshot of memcached, and the 1.0.8 PHP API, and > >>the latest PHP 4.3.3 CVS tree, under Linux 2.4.21, Apache 2.0.47. It was > >>all working great on my devel box, but when I moved it over to production, I > >>started getting a ton of "_load_items(): Bytes read is greater than > >>requested data size." errors. But the thing is: the development box and > >>production box are using the exact same memcached server. They run the same > >>OS, the same PHP version, the same *everything*. They're even the same > >>hardware (one serial number apart). The *only* difference is the production > >>box is on the same subnet as the memcached server -- the development one is > >>on a different one. Of course, I start doing a lot more testing on the > >>development box, and I notice I was getting them over there too, just not as > >>many. > >> > >>So I modified the script to give me more information (patch follows): > >> > >>================= > >>1063,1064c1063,1064 > >>< $this->_debug("_load_items(): Bytes > >>read is greater than requested data size."); > >>< > >>--- > >> > >>> $this->_debug("_load_items(): Bytes > >> > >>read ($bytes_read) is greater than requested data size ($len)."); > >> > >>> $this->_debug("_load_items(): I > >> > >>received \"".urlencode($buf)."\""); > >>================= > >> > >>Here's what I'm getting in my logs (I modified the debug handler to use my > >>custom logging routines) -- add 7 for the "\r\nEND\r\n". So we're getting > >>back 2 characters instead of 1: > >> > >>(Data is "0") > >>[03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes > >>read (9) is greater than requested data size (1). > >>[03Aug12 154857] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I > >>received "%0A2%0D%0AEND%0D%0A" > >> > >>(Data is "-1") > >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes > >>read (10) is greater than requested data size (2). > >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I > >>received "%0A-1%0D%0AEND%0D%0A" > >> > >>(Data is a whole lot more than "0" :) ): > >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): Bytes > >>read (165) is greater than requested data size (157). > >>[03Aug12 154926] (INFO) #/profile/# <10.1.0.11> (u:1) _load_items(): I > >>received "%0Aa%3A1%3A%7Bi%3A0%3Ba%3A5%3A%7Bs%3A7%3A%(for clarity, I cut off > >>the end) > >> > >>The thing that sucks is I can't replicate this reliably. I can hit reload > >>over and over, and it will only happen 1 out of every 10 or so times, on > >>random data coming back. It's not just numerics, not text... > >> > >>I sniffed the connection, and here's one that barfed coming back from the > >>server: > >> > >>VALUE profile_1_7028277b0e9405414ea71de0bdde789f_rank 0 2\r\n-1\r\nEND\r\n > >> > >>Looks okay to me; it follows the protocol, and there are two characters in > >>the response. Memcached isn't the culprit. So on to the API... > >> > >>The problem appears to be with this line: > >>$line = substr($line, strpos($line, "\r\n")+1, strlen($line)); > >> > >>I added some more logging lines before and after it to see what the 'pre' > >>and 'post' parsing results were. > >> > >>By only doing +1, you're just stripping off the \r. It's always letting a > >>\n through, even though it should be stripping it off. The strange thing to > >>me is sometimes it only lets a \n through, but sometimes it lets > >>"\n0\r\nEND\r\n" through (this is when it breaks). I'm still trying to > >>track down why this is happening... > >> > >>In summary; I changed that +1 to a +2, and it always strips off the > >>remaining \n. From there, it works great. I just pushed about 300 pages, > >>and all came back without errors... > >> > >>Final patch: > >>============== > >>999c999 > >>< $line = substr($line, strpos($line, > >>"\r\n")+1, strlen($line)); > >>--- > >> > >>> $line = substr($line, strpos($line, > >> > >>"\r\n")+2, strlen($line)); > >>=============== > >> > >>"It works for me". :) > >> > >>Justin > >> > >>----- Original Message ----- > >>From: "Ryan Gilfether" > >>To: > >>Sent: Monday, August 11, 2003 12:37 AM > >>Subject: Re: PHP API 1.0.7 > >> > >> > >> > >>>yes, you're right, I didnt notice that a newline got included at the end > >>>of the file. I'll be sure to remove it in future releases. Thanks for > >>>pointing it out. > >>> > >>>Ryan > >>> > >>>Lisa Marie Seelye wrote: > >>> > >>>>On Sat, 2003-08-09 at 13:08, Ryan Gilfether wrote: > >>>> > >>>> > >>>>>A new version of the PHP API has been completed, verson 1.0.7. > >>>>>I've had some requests for adding PHPDocs style comments to the API. > >>>>>Along with that 3 new function have been added > >>>>>error() - returns an error code for the last error genterated > >>>>>error_string() - returns a string with a description of the error > >>>>>error_clear() - clears the last error > >>>>> > >>>>>Ryan > >>>> > >>>> > >>>>Not to be incredibly anal, but the .php file should not have any > >>>>newlines after the closing ?> tag -- this tends to mess with headers > >>>>should it be included after headers have been sent. :( > >>>> > >>>> > >>> > >>>-- > >>>Ryan Gilfether > >>>Software Engineer > >>>http://www.gilfether.com > >>> > >>> > >> > >> > > > > > > -- > Ryan Gilfether > Software Engineer > http://www.gilfether.com > From lisa@gentoo.org Wed Aug 13 00:57:58 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Tue, 12 Aug 2003 19:57:58 -0400 Subject: possible bug in PHP API 1.0.8 In-Reply-To: <3F39775D.6070309@gilfether.com> References: <3F352A7D.9090306@gilfether.com> <1060574543.1832.9.camel@lisa.thedoh.com> <3F371D94.5040100@gilfether.com> <002701c36110$e0584c80$0b00010a@nomium.net> <3F39775D.6070309@gilfether.com> Message-ID: <1060732678.1820.16.camel@lisa.thedoh.com> --=-U++ZkE9eIAOHgGqD2Dg6 Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Tue, 2003-08-12 at 19:25, Ryan Gilfether wrote: The change only change necessary was the +1 to +2. --=20 Regards, -Lisa --=-U++ZkE9eIAOHgGqD2Dg6 Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/OX8Gy0a1Vh5Jb8URAs5sAJ440sqyccetjUgKDI71JskbXlBY9gCgxJgM 420h5pq2axbdd7A4vIOfAmg= =frmk -----END PGP SIGNATURE----- --=-U++ZkE9eIAOHgGqD2Dg6-- From brad@danga.com Wed Aug 13 01:04:25 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 17:04:25 -0700 (PDT) Subject: PHP API 1.0.9 Message-ID: New PHP release is up, with the fix for the protocol parsing bug. http://www.danga.com/memcached/dist/php-memcached-1.0.9.tar.gz - Brad From brad@danga.com Wed Aug 13 01:38:10 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Tue, 12 Aug 2003 17:38:10 -0700 (PDT) Subject: new server snapshot Message-ID: Here's another pre-1.1.9 snapshot: http://www.danga.com/memcached/dist/snapshots/memcached-1.1.9-snapshot.tar.gz We're now running it on one LiveJourna machine and it seems fine. The changes are really minimal from 1.1.8. Changes since last snapshot: * It's much faster, using TCP_CORK on Linux, TCP_NOPUSH on BSD, and TCP_NODELAY everywhere else. Change since 1.1.8: * Portability fixes for BSD/OS X. Known issues: * autoconf needs to --disable-static on OS X, because Darwin doesn't support static binaries. - Brad From mellon@pobox.com Wed Aug 13 06:48:49 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Wed, 13 Aug 2003 08:48:49 +0300 Subject: network performance measurements In-Reply-To: References: <20030812232350.GA24758@pobox.com> Message-ID: <20030813054849.GA25249@pobox.com> On Tue, Aug 12, 2003 at 04:43:49PM -0700, Brad Fitzpatrick wrote: > I was thinking: how about I just ditch c->write_and_uncork and check > instead of c->write_and_go == conn_read? If the state machine is moving > from writing to reading, it's done writing. > > I'll do that. Reply if you can think of a case where that's the wrong > thing, let me know. No, I can't; that seems safe enough. -- avva From mellon@pobox.com Wed Aug 13 10:46:22 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Wed, 13 Aug 2003 12:46:22 +0300 Subject: new server snapshot In-Reply-To: References: Message-ID: <20030813094622.GA25731@pobox.com> On Tue, Aug 12, 2003 at 05:38:10PM -0700, Brad Fitzpatrick wrote: > * It's much faster, using TCP_CORK on Linux, TCP_NOPUSH on BSD, and > TCP_NODELAY everywhere else. So what do you want to do with the write-combining code I sent the other day? I can make it refrain from combining writes if TCP_CORK is on, but do it if TCP_NODELAY is used... seems to be the best option, no? > * autoconf needs to --disable-static on OS X, because Darwin doesn't > support static binaries. What, no static binaries at all? Amazing. Also, this little patch is almost cosmetic, but it moves the uncorking to the right place, from the code design point of view. main <- wcmtools src/memcached/memcached.c --- src/memcached/memcached.c 2003-08-13 12:36:23.000000000 +0300 +++ cvs/wcmtools/memcached/memcached.c 2003-08-13 12:39:35.000000000 +0300 @@ -933,6 +933,8 @@ c->write_and_free = 0; } c->state = c->write_and_go; + if (c->state == conn_read) + set_cork(c, 0); break; } res = write(c->sfd, c->wcurr, c->wbytes); @@ -940,8 +942,6 @@ stats.bytes_written += res; c->wcurr += res; c->wbytes -= res; - if (c->wbytes == 0 && c->write_and_go == conn_read) - set_cork(c, 0); break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { From brad@danga.com Wed Aug 13 19:39:37 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 11:39:37 -0700 (PDT) Subject: new server snapshot In-Reply-To: <20030813094622.GA25731@pobox.com> References: <20030813094622.GA25731@pobox.com> Message-ID: I wanted to uncork as early as possible, right after the write, not when we get back into the state machine. Because how do we get back into the drive_machine? When some event occurs on our socket. What event? Finishing writing? We don't want that delay. If anything, why don't we move the c->wbytes == 0 down below after the real write? I don't like doing half the work in one cycle of the state machine and doing the rest later. It was bugging me, so I asked Evan if I was misunderstanding non-blocking writes, like whether the buffer in use until it's fully written, but Evan said it's still copied into a buffer, so we could do the free() after the successful write(). - Brad On Wed, 13 Aug 2003, Anatoly Vorobey wrote: > On Tue, Aug 12, 2003 at 05:38:10PM -0700, Brad Fitzpatrick wrote: > > * It's much faster, using TCP_CORK on Linux, TCP_NOPUSH on BSD, and > > TCP_NODELAY everywhere else. > > So what do you want to do with the write-combining code I sent the > other day? I can make it refrain from combining writes if TCP_CORK > is on, but do it if TCP_NODELAY is used... seems to be the best option, > no? > > > * autoconf needs to --disable-static on OS X, because Darwin doesn't > > support static binaries. > > What, no static binaries at all? Amazing. > > Also, this little patch is almost cosmetic, but it moves the uncorking > to the right place, from the code design point of view. > > main <- wcmtools src/memcached/memcached.c > --- src/memcached/memcached.c 2003-08-13 12:36:23.000000000 +0300 > +++ cvs/wcmtools/memcached/memcached.c 2003-08-13 12:39:35.000000000 +0300 > @@ -933,6 +933,8 @@ > c->write_and_free = 0; > } > c->state = c->write_and_go; > + if (c->state == conn_read) > + set_cork(c, 0); > break; > } > res = write(c->sfd, c->wcurr, c->wbytes); > @@ -940,8 +942,6 @@ > stats.bytes_written += res; > c->wcurr += res; > c->wbytes -= res; > - if (c->wbytes == 0 && c->write_and_go == conn_read) > - set_cork(c, 0); > break; > } > if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { > > From brad@danga.com Wed Aug 13 19:44:46 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 11:44:46 -0700 (PDT) Subject: Compression: client or server? Message-ID: So far the two big users of memcache (LiveJournal and Slashdot) are using it to store text, and lots of text. Text compresses very well. Large chunks of text compress better, but little text still compresses quite well. So, memcache should support compression. But, where to do it? Any client can already do it without server knowledge, using one of the 16 opaque flags. The Perl client could, say, check for items larger than 1k and try to compress it using Compress::Zlib, and if compression helps, set a bit meaning "I'm compressed!", and it'll uncompress it later. Pro: no server support. compression CPU is distributed. Con: the server can't report stats about total usage vs. compressed usage, which would be neat. Or, the server could do it: Pro: stats. Con: CPU drain. Or, what if we augmented the protocol to allow a client to report both the real size and the compressed size? Pro: CPU still on clients. server can do stats. Con: protocol change (reverse compatible, though), and an extra 4 bytes per compressed item, to store the other size. Thoughts? - Brad From lisa@gentoo.org Wed Aug 13 20:00:12 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Wed, 13 Aug 2003 15:00:12 -0400 Subject: Compression: client or server? In-Reply-To: References: Message-ID: <1060801212.1753.2.camel@lisa.thedoh.com> --=-RBWbODTdK3WxLQkDgDVu Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > Thoughts? What CPU resources are required to compress the data? How much of a lag will it introduce to the System? The best way to go about storing compressed data is for the client to _send the data in a compressed format_ and then know well enough to decompress it when they get it back. The server should store whatever the client sends them and don't mess with it -- put the burden on the user, not the server. --=20 Regards, -Lisa --=-RBWbODTdK3WxLQkDgDVu Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/Ooq8y0a1Vh5Jb8URAhbCAJ96z76DZ/CWIX2b3h7c8Ka03PtnggCgm4Ma Q15H0+Rn9UsYNBd/8Rmi/CM= =JRQc -----END PGP SIGNATURE----- --=-RBWbODTdK3WxLQkDgDVu-- From mellon@pobox.com Wed Aug 13 20:06:27 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Wed, 13 Aug 2003 22:06:27 +0300 Subject: new server snapshot In-Reply-To: References: <20030813094622.GA25731@pobox.com> Message-ID: <20030813190627.GA28827@pobox.com> You wrote on Wed, Aug 13, 2003 at 11:39:37AM -0700: > I wanted to uncork as early as possible, right after the write, not when > we get back into the state machine. Because how do we get back into the > drive_machine? When some event occurs on our socket. What event? > Finishing writing? We don't want that delay. Err, no. We stay in the state machine as long as possible. After we write(), we break and immediately get back into the same case clause (since machine state didn't change, and exit flag wasn't set), and check whether wbytes==0. The only way we're getting out of the state machine is via that if(wbytes==0) (in fact, even when we're not getting out, we're changing states), or when one of our write()'s block. The delay is all of 3 CPU instructions or something. That's the general philosophy of that function: do something atomic and enter the state machine (via "break", which leads to the beginning of the loop) again, sometimes entering the same case (if you didn't change the state), sometimes not. Your actions inside the state machine don't call one another directly, they cause one another by changing state or other variables (like wbytes). The delay caused by doing a break; which immediately leads from the top-level switch to the same code, versus wrapping such an action in an extra loop, is absolutely negligible, especially considering the CPU cache, and the code is better designed and easier to change. The reason to move the uncorking inside the if(wbytes==0) clause is that this is the clause for exiting the write state. If we had, for some reason, dynamic response strings which could by 0 bytes long (no response to some command), entering the write state with wbytes==0 would take us right to the if(wbytes==0) clause and we'd miss the uncorking. Since we don't actually have that situation occurring, it's more of a cosmetic patch (as I wrote), but this example shows why this is the right place for it. -- avva From brad@danga.com Wed Aug 13 20:53:20 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 12:53:20 -0700 (PDT) Subject: Compression: client or server? In-Reply-To: <1060801212.1753.2.camel@lisa.thedoh.com> References: <1060801212.1753.2.camel@lisa.thedoh.com> Message-ID: I agree. The issue then is stats, though. It'd be interesting to know how much physical vs. logical data is in memcache. Based on that, you could adjust the compression threshold on the clients. I imagine adding an option to the Perl MemCachedClient API that says, "Compress after this size". At lunch just now Evan and I realized another point to add it to the server, though: what if somebody had more memcache servers than web nodes? But we don't buy this argument, because in our experience you can't just _buy_ a memcache machine. We were unable to find a system or parts vendor who could get us a machine with a crap processor and tons of RAM. So, yeah: compression definitely client-side, but the server should do stats by the client sending more data about how much it shrank it to. See any problem with that? On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > Thoughts? > > What CPU resources are required to compress the data? How much of a lag > will it introduce to the System? > > The best way to go about storing compressed data is for the client to > _send the data in a compressed format_ and then know well enough to > decompress it when they get it back. > > The server should store whatever the client sends them and don't mess > with it -- put the burden on the user, not the server. > > > -- > Regards, > -Lisa > > From brad@danga.com Wed Aug 13 20:56:00 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 12:56:00 -0700 (PDT) Subject: new server snapshot In-Reply-To: <20030813190627.GA28827@pobox.com> References: <20030813094622.GA25731@pobox.com> <20030813190627.GA28827@pobox.com> Message-ID: Ah! while(!exit) { ... } I didn't see the while. I was thinking there'd be a huge delay because we'd exist drive_machine and then wait for some event to come back into it. Will apply the cosmetic patch. On Wed, 13 Aug 2003, Anatoly Vorobey wrote: > You wrote on Wed, Aug 13, 2003 at 11:39:37AM -0700: > > I wanted to uncork as early as possible, right after the write, not when > > we get back into the state machine. Because how do we get back into the > > drive_machine? When some event occurs on our socket. What event? > > Finishing writing? We don't want that delay. > > Err, no. We stay in the state machine as long as possible. After > we write(), we break and immediately get back into the same case > clause (since machine state didn't change, and exit flag wasn't set), > and check whether wbytes==0. The only way we're getting out > of the state machine is via that if(wbytes==0) (in fact, even when > we're not getting out, we're changing states), or when one of our > write()'s block. > > The delay is all of 3 CPU instructions or something. > > That's the general philosophy of that function: do something atomic > and enter the state machine (via "break", which leads to the beginning > of the loop) again, sometimes entering the same case (if you didn't > change the state), sometimes not. Your actions inside the state > machine don't call one another directly, they cause one another by > changing state or other variables (like wbytes). The delay caused by > doing a break; which immediately leads from the top-level switch > to the same code, versus wrapping such an action in an extra loop, > is absolutely negligible, especially considering the CPU cache, and > the code is better designed and easier to change. > > The reason to move the uncorking inside the if(wbytes==0) clause is > that this is the clause for exiting the write state. If we had, for > some reason, dynamic response strings which could by 0 bytes long (no > response to some command), entering the write state with wbytes==0 > would take us right to the if(wbytes==0) clause and we'd miss the > uncorking. Since we don't actually have that situation occurring, it's > more of a cosmetic patch (as I wrote), but this example shows why this > is the right place for it. > > -- > avva > > From lisa@gentoo.org Wed Aug 13 21:07:30 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Wed, 13 Aug 2003 16:07:30 -0400 Subject: Compression: client or server? In-Reply-To: References: <1060801212.1753.2.camel@lisa.thedoh.com> Message-ID: <1060805250.1753.13.camel@lisa.thedoh.com> --=-UPrHodqtDKkD/9Ha+4JP Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Wed, 2003-08-13 at 15:53, Brad Fitzpatrick wrote: > I agree. >=20 > The issue then is stats, though. It'd be interesting to know how much > physical vs. logical data is in memcache. Based on that, you could adjus= t > the compression threshold on the clients. I imagine adding an option to > the Perl MemCachedClient API that says, "Compress after this size". This is a good idea for the API, but not really relevant to the server, as long as the server keeps those flags intact!! > At lunch just now Evan and I realized another point to add it to the > server, though: what if somebody had more memcache servers than web > nodes? But we don't buy this argument, because in our experience you > can't just _buy_ a memcache machine. We were unable to find a system or > parts vendor who could get us a machine with a crap processor and tons of > RAM. >=20 > So, yeah: compression definitely client-side, but the server should do > stats by the client sending more data about how much it shrank it to. Se= e > any problem with that? I don't see what difference it makes to the server if all the data is data. Whether or not its a Shakespearean play or a compressed jpeg what should the server care? --=20 Regards, -Lisa --=-UPrHodqtDKkD/9Ha+4JP Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/OpqCy0a1Vh5Jb8URAhD4AKCVVuwQ+5GFWlKmbxeeGlKzXHOpUwCdH0QG opqf9fTcn5xUQxaewbknq3M= =KwUj -----END PGP SIGNATURE----- --=-UPrHodqtDKkD/9Ha+4JP-- From jmat@shutdown.net Wed Aug 13 22:24:15 2003 From: jmat@shutdown.net (Justin Matlock) Date: Wed, 13 Aug 2003 17:24:15 -0400 Subject: Compression: client or server? References: <1060801212.1753.2.camel@lisa.thedoh.com> Message-ID: <005001c361e1$3f9b8b00$0b00010a@nomium.net> This is a multi-part message in MIME format. ------=_NextPart_000_004D_01C361BF.B815CC70 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: 7bit You guys read my mind. :) I've been doing this for a few days now... in tests, it takes 400k text (the linux words file), and takes it down to 100k. The extra time it took to compress (a few milliseconds) was insignificant compared to the speed increase from the size reduction. As far as CPU load goes -- I didn't notice a major increase in CPU, but it's somewhat hard to measure with PHP. My client machine is a Linux box on a AMD Athlon 700, 256mb RAM, so your results may vary. Attached patch to make the PHP API use gzip compression; it also updates the Documentation file about the new $options['compress'] configuration variable, which allows complete disabling of compression, or setting a minimum threshold to activate compression (I recommend 10240 from testing). Justin ----- Original Message ----- From: "Brad Fitzpatrick" To: Sent: Wednesday, August 13, 2003 3:53 PM Subject: Re: Compression: client or server? > I agree. > > The issue then is stats, though. It'd be interesting to know how much > physical vs. logical data is in memcache. Based on that, you could adjust > the compression threshold on the clients. I imagine adding an option to > the Perl MemCachedClient API that says, "Compress after this size". > > At lunch just now Evan and I realized another point to add it to the > server, though: what if somebody had more memcache servers than web > nodes? But we don't buy this argument, because in our experience you > can't just _buy_ a memcache machine. We were unable to find a system or > parts vendor who could get us a machine with a crap processor and tons of > RAM. > > So, yeah: compression definitely client-side, but the server should do > stats by the client sending more data about how much it shrank it to. See > any problem with that? > > > On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > > > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > > Thoughts? > > > > What CPU resources are required to compress the data? How much of a lag > > will it introduce to the System? > > > > The best way to go about storing compressed data is for the client to > > _send the data in a compressed format_ and then know well enough to > > decompress it when they get it back. > > > > The server should store whatever the client sends them and don't mess > > with it -- put the burden on the user, not the server. > > > > > > -- > > Regards, > > -Lisa > > > > > ------=_NextPart_000_004D_01C361BF.B815CC70 Content-Type: application/octet-stream; name="php-memcached-gzip.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="php-memcached-gzip.patch" diff -f php-memcached-1.0.9/Documentation = php-memcached-1.0.9gzip/Documentation=0A= a20=0A= // $options["compress"] controls gzip compression; set to 0 to completely=0A= // disable, set to above zero to set the compression threshold: if the=0A= // data is smaller than this amount (in bytes), don't compress, if it's=0A= // larger, compress.=0A= .=0A= diff -f php-memcached-1.0.9/MemCachedClient.inc.php = php-memcached-1.0.9gzip/MemCachedClient.inc.php=0A= a92=0A= /**=0A= * size of val to force compression; 0 turns off=0A= * @ var int=0A= */=0A= var $compress; =0A= .=0A= a118=0A= $this->compress =3D $options["compress"];=0A= .=0A= d786=0A= a839=0A= if (($this->compress > 0) && (strlen($val) > $this->compress)) {=0A= $this->_debug("_set(): compressing data. size in:".strlen($val));=0A= $val=3Dgzcompress($val);=0A= $this->_debug("_set(): done compressing data. size = out:".strlen($val));=0A= $flags |=3D 2;=0A= }=0A= =0A= .=0A= a1036=0A= if(@$flags_array[$sk] & 2)=0A= $val[$sk] =3D gzuncompress($val[$sk],$len);=0A= .=0A= ------=_NextPart_000_004D_01C361BF.B815CC70-- From brad@danga.com Wed Aug 13 22:27:34 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 14:27:34 -0700 (PDT) Subject: Compression: client or server? In-Reply-To: <005001c361e1$3f9b8b00$0b00010a@nomium.net> References: <1060801212.1753.2.camel@lisa.thedoh.com> <005001c361e1$3f9b8b00$0b00010a@nomium.net> Message-ID: Damn, the PHP client is getting ahead!? I must update the Perl client! What's with this cracked-out patch format, though? Add to your ~/.cvsrc this: cvs -z3 diff -u update -dP checkout -A On Wed, 13 Aug 2003, Justin Matlock wrote: > You guys read my mind. :) > > I've been doing this for a few days now... in tests, it takes 400k text > (the linux words file), and takes it down to 100k. The extra time it took > to compress (a few milliseconds) was insignificant compared to the speed > increase from the size reduction. As far as CPU load goes -- I didn't > notice a major increase in CPU, but it's somewhat hard to measure with PHP. > My client machine is a Linux box on a AMD Athlon 700, 256mb RAM, so your > results may vary. > > Attached patch to make the PHP API use gzip compression; it also updates the > Documentation file about the new $options['compress'] configuration > variable, which allows complete disabling of compression, or setting a > minimum threshold to activate compression (I recommend 10240 from testing). > > Justin > > > ----- Original Message ----- > From: "Brad Fitzpatrick" > To: > Sent: Wednesday, August 13, 2003 3:53 PM > Subject: Re: Compression: client or server? > > > > I agree. > > > > The issue then is stats, though. It'd be interesting to know how much > > physical vs. logical data is in memcache. Based on that, you could adjust > > the compression threshold on the clients. I imagine adding an option to > > the Perl MemCachedClient API that says, "Compress after this size". > > > > At lunch just now Evan and I realized another point to add it to the > > server, though: what if somebody had more memcache servers than web > > nodes? But we don't buy this argument, because in our experience you > > can't just _buy_ a memcache machine. We were unable to find a system or > > parts vendor who could get us a machine with a crap processor and tons of > > RAM. > > > > So, yeah: compression definitely client-side, but the server should do > > stats by the client sending more data about how much it shrank it to. See > > any problem with that? > > > > > > On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > > > > > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > > > Thoughts? > > > > > > What CPU resources are required to compress the data? How much of a lag > > > will it introduce to the System? > > > > > > The best way to go about storing compressed data is for the client to > > > _send the data in a compressed format_ and then know well enough to > > > decompress it when they get it back. > > > > > > The server should store whatever the client sends them and don't mess > > > with it -- put the burden on the user, not the server. > > > > > > > > > -- > > > Regards, > > > -Lisa > > > > > > > > > From jmat@shutdown.net Wed Aug 13 22:36:06 2003 From: jmat@shutdown.net (Justin Matlock) Date: Wed, 13 Aug 2003 17:36:06 -0400 Subject: Compression: client or server? References: <1060801212.1753.2.camel@lisa.thedoh.com> <005001c361e1$3f9b8b00$0b00010a@nomium.net> Message-ID: <006101c361e2$e742be40$0b00010a@nomium.net> This is a multi-part message in MIME format. ------=_NextPart_000_005E_01C361C1.5FF72280 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: 7bit Doh. Corrected patch. gzipuncompress was failing in very rare cases (null on the end of the compressed data stream). If you don't want to patch, just remove the ",$len" from the end of the gzuncompress line. :) ----- Original Message ----- From: "Justin Matlock" To: Sent: Wednesday, August 13, 2003 5:24 PM Subject: Re: Compression: client or server? > You guys read my mind. :) > > I've been doing this for a few days now... in tests, it takes 400k text > (the linux words file), and takes it down to 100k. The extra time it took > to compress (a few milliseconds) was insignificant compared to the speed > increase from the size reduction. As far as CPU load goes -- I didn't > notice a major increase in CPU, but it's somewhat hard to measure with PHP. > My client machine is a Linux box on a AMD Athlon 700, 256mb RAM, so your > results may vary. > > Attached patch to make the PHP API use gzip compression; it also updates the > Documentation file about the new $options['compress'] configuration > variable, which allows complete disabling of compression, or setting a > minimum threshold to activate compression (I recommend 10240 from testing). > > Justin > > > ----- Original Message ----- > From: "Brad Fitzpatrick" > To: > Sent: Wednesday, August 13, 2003 3:53 PM > Subject: Re: Compression: client or server? > > > > I agree. > > > > The issue then is stats, though. It'd be interesting to know how much > > physical vs. logical data is in memcache. Based on that, you could adjust > > the compression threshold on the clients. I imagine adding an option to > > the Perl MemCachedClient API that says, "Compress after this size". > > > > At lunch just now Evan and I realized another point to add it to the > > server, though: what if somebody had more memcache servers than web > > nodes? But we don't buy this argument, because in our experience you > > can't just _buy_ a memcache machine. We were unable to find a system or > > parts vendor who could get us a machine with a crap processor and tons of > > RAM. > > > > So, yeah: compression definitely client-side, but the server should do > > stats by the client sending more data about how much it shrank it to. See > > any problem with that? > > > > > > On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > > > > > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > > > Thoughts? > > > > > > What CPU resources are required to compress the data? How much of a lag > > > will it introduce to the System? > > > > > > The best way to go about storing compressed data is for the client to > > > _send the data in a compressed format_ and then know well enough to > > > decompress it when they get it back. > > > > > > The server should store whatever the client sends them and don't mess > > > with it -- put the burden on the user, not the server. > > > > > > > > > -- > > > Regards, > > > -Lisa > > > > > > > > > ------=_NextPart_000_005E_01C361C1.5FF72280 Content-Type: application/octet-stream; name="php-memcached-gzip2.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="php-memcached-gzip2.patch" diff -f php-memcached-1.0.9/Documentation = php-memcached-1.0.9gzip/Documentation=0A= a20=0A= // $options["compress"] controls gzip compression; set to 0 to completely=0A= // disable, set to above zero to set the compression threshold: if the=0A= // data is smaller than this amount (in bytes), don't compress, if it's=0A= // larger, compress.=0A= .=0A= diff -f php-memcached-1.0.9/MemCachedClient.inc.php = php-memcached-1.0.9gzip/MemCachedClient.inc.php=0A= a92=0A= /**=0A= * size of val to force compression; 0 turns off=0A= * @ var int=0A= */=0A= var $compress; =0A= .=0A= a118=0A= $this->compress =3D $options["compress"];=0A= .=0A= d786=0A= a839=0A= if (($this->compress > 0) && (strlen($val) > $this->compress)) {=0A= $this->_debug("_set(): compressing data. size in:".strlen($val));=0A= $val=3Dgzcompress($val);=0A= $this->_debug("_set(): done compressing data. size = out:".strlen($val));=0A= $flags |=3D 2;=0A= }=0A= =0A= .=0A= a1036=0A= if(@$flags_array[$sk] & 2)=0A= $val[$sk] =3D gzuncompress($val[$sk]);=0A= .=0A= ------=_NextPart_000_005E_01C361C1.5FF72280-- From brad@danga.com Wed Aug 13 22:38:23 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 14:38:23 -0700 (PDT) Subject: Compression: client or server? In-Reply-To: <006101c361e2$e742be40$0b00010a@nomium.net> References: <1060801212.1753.2.camel@lisa.thedoh.com> <005001c361e1$3f9b8b00$0b00010a@nomium.net> <006101c361e2$e742be40$0b00010a@nomium.net> Message-ID: I only like unified diffs (-u). Please post those instead. On Wed, 13 Aug 2003, Justin Matlock wrote: > Doh. Corrected patch. gzipuncompress was failing in very rare cases (null > on the end of the compressed data stream). > > If you don't want to patch, just remove the ",$len" from the end of the > gzuncompress line. :) > > ----- Original Message ----- > From: "Justin Matlock" > To: > Sent: Wednesday, August 13, 2003 5:24 PM > Subject: Re: Compression: client or server? > > > > You guys read my mind. :) > > > > I've been doing this for a few days now... in tests, it takes 400k text > > (the linux words file), and takes it down to 100k. The extra time it took > > to compress (a few milliseconds) was insignificant compared to the speed > > increase from the size reduction. As far as CPU load goes -- I didn't > > notice a major increase in CPU, but it's somewhat hard to measure with > PHP. > > My client machine is a Linux box on a AMD Athlon 700, 256mb RAM, so your > > results may vary. > > > > Attached patch to make the PHP API use gzip compression; it also updates > the > > Documentation file about the new $options['compress'] configuration > > variable, which allows complete disabling of compression, or setting a > > minimum threshold to activate compression (I recommend 10240 from > testing). > > > > Justin > > > > > > ----- Original Message ----- > > From: "Brad Fitzpatrick" > > To: > > Sent: Wednesday, August 13, 2003 3:53 PM > > Subject: Re: Compression: client or server? > > > > > > > I agree. > > > > > > The issue then is stats, though. It'd be interesting to know how much > > > physical vs. logical data is in memcache. Based on that, you could > adjust > > > the compression threshold on the clients. I imagine adding an option to > > > the Perl MemCachedClient API that says, "Compress after this size". > > > > > > At lunch just now Evan and I realized another point to add it to the > > > server, though: what if somebody had more memcache servers than web > > > nodes? But we don't buy this argument, because in our experience you > > > can't just _buy_ a memcache machine. We were unable to find a system or > > > parts vendor who could get us a machine with a crap processor and tons > of > > > RAM. > > > > > > So, yeah: compression definitely client-side, but the server should do > > > stats by the client sending more data about how much it shrank it to. > See > > > any problem with that? > > > > > > > > > On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > > > > > > > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > > > > Thoughts? > > > > > > > > What CPU resources are required to compress the data? How much of a > lag > > > > will it introduce to the System? > > > > > > > > The best way to go about storing compressed data is for the client to > > > > _send the data in a compressed format_ and then know well enough to > > > > decompress it when they get it back. > > > > > > > > The server should store whatever the client sends them and don't mess > > > > with it -- put the burden on the user, not the server. > > > > > > > > > > > > -- > > > > Regards, > > > > -Lisa > > > > > > > > > > > > > > From jmat@shutdown.net Wed Aug 13 22:43:35 2003 From: jmat@shutdown.net (Justin Matlock) Date: Wed, 13 Aug 2003 17:43:35 -0400 Subject: Compression: client or server? References: <1060801212.1753.2.camel@lisa.thedoh.com> <005001c361e1$3f9b8b00$0b00010a@nomium.net> <006101c361e2$e742be40$0b00010a@nomium.net> Message-ID: <007101c361e3$f2b2cb20$0b00010a@nomium.net> This is a multi-part message in MIME format. ------=_NextPart_000_006E_01C361C2.6B6D22D0 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: 7bit Ugh; sorry.. I don't know what the heck that was... Not sleeping for two days'll do that. :) ----- Original Message ----- From: "Brad Fitzpatrick" To: Sent: Wednesday, August 13, 2003 5:38 PM Subject: Re: Compression: client or server? > I only like unified diffs (-u). Please post those instead. > > > On Wed, 13 Aug 2003, Justin Matlock wrote: > > > Doh. Corrected patch. gzipuncompress was failing in very rare cases (null > > on the end of the compressed data stream). > > > > If you don't want to patch, just remove the ",$len" from the end of the > > gzuncompress line. :) > > > > ----- Original Message ----- > > From: "Justin Matlock" > > To: > > Sent: Wednesday, August 13, 2003 5:24 PM > > Subject: Re: Compression: client or server? > > > > > > > You guys read my mind. :) > > > > > > I've been doing this for a few days now... in tests, it takes 400k text > > > (the linux words file), and takes it down to 100k. The extra time it took > > > to compress (a few milliseconds) was insignificant compared to the speed > > > increase from the size reduction. As far as CPU load goes -- I didn't > > > notice a major increase in CPU, but it's somewhat hard to measure with > > PHP. > > > My client machine is a Linux box on a AMD Athlon 700, 256mb RAM, so your > > > results may vary. > > > > > > Attached patch to make the PHP API use gzip compression; it also updates > > the > > > Documentation file about the new $options['compress'] configuration > > > variable, which allows complete disabling of compression, or setting a > > > minimum threshold to activate compression (I recommend 10240 from > > testing). > > > > > > Justin > > > > > > > > > ----- Original Message ----- > > > From: "Brad Fitzpatrick" > > > To: > > > Sent: Wednesday, August 13, 2003 3:53 PM > > > Subject: Re: Compression: client or server? > > > > > > > > > > I agree. > > > > > > > > The issue then is stats, though. It'd be interesting to know how much > > > > physical vs. logical data is in memcache. Based on that, you could > > adjust > > > > the compression threshold on the clients. I imagine adding an option to > > > > the Perl MemCachedClient API that says, "Compress after this size". > > > > > > > > At lunch just now Evan and I realized another point to add it to the > > > > server, though: what if somebody had more memcache servers than web > > > > nodes? But we don't buy this argument, because in our experience you > > > > can't just _buy_ a memcache machine. We were unable to find a system or > > > > parts vendor who could get us a machine with a crap processor and tons > > of > > > > RAM. > > > > > > > > So, yeah: compression definitely client-side, but the server should do > > > > stats by the client sending more data about how much it shrank it to. > > See > > > > any problem with that? > > > > > > > > > > > > On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > > > > > > > > > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > > > > > Thoughts? > > > > > > > > > > What CPU resources are required to compress the data? How much of a > > lag > > > > > will it introduce to the System? > > > > > > > > > > The best way to go about storing compressed data is for the client to > > > > > _send the data in a compressed format_ and then know well enough to > > > > > decompress it when they get it back. > > > > > > > > > > The server should store whatever the client sends them and don't mess > > > > > with it -- put the burden on the user, not the server. > > > > > > > > > > > > > > > -- > > > > > Regards, > > > > > -Lisa > > > > > > > > > > > > > > > > > > > > ------=_NextPart_000_006E_01C361C2.6B6D22D0 Content-Type: application/octet-stream; name="php-memcached-1.0.9.gzip.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="php-memcached-1.0.9.gzip.patch" --- php-memcached-1.0.9/MemCachedClient.inc.php Tue Aug 12 20:09:53 2003=0A= +++ php-memcached-1.0.9.gzip/MemCachedClient.inc.php Wed Aug 13 17:39:02 = 2003=0A= @@ -90,6 +90,11 @@=0A= * @var string=0A= */=0A= var $errstr;=0A= + /**=0A= + * size of val to force compression; 0 turns off=0A= + * @ var int=0A= + */=0A= + var $compress;=0A= =0A= =0A= /**=0A= @@ -116,6 +121,7 @@=0A= {=0A= $this->set_servers($options["servers"]);=0A= $this->debug =3D $options["debug"];=0A= + $this->compress =3D $options["compress"];=0A= $this->cache_sock =3D array();=0A= }=0A= =0A= @@ -783,7 +789,6 @@=0A= return trim($retval);=0A= }=0A= =0A= -=0A= /**=0A= * sends the command to the server=0A= * Possible errors set are:=0A= @@ -837,6 +842,13 @@=0A= $flags |=3D 1;=0A= }=0A= =0A= + if (($this->compress > 0) && (strlen($val) > $this->compress)) {=0A= + $this->_debug("_set(): compressing data. size in:".strlen($val));=0A= + $val=3Dgzcompress($val);=0A= + $this->_debug("_set(): done compressing data. size = out:".strlen($val));=0A= + $flags |=3D 2;=0A= + }=0A= +=0A= $len =3D strlen($val);=0A= if (!is_int($exptime))=0A= $exptime =3D 0;=0A= @@ -1034,6 +1046,8 @@=0A= =0A= if(strlen($val[$sk]) !=3D $len_array[$sk])=0A= continue;=0A= + if(@$flags_array[$sk] & 2)=0A= + $val[$sk] =3D gzuncompress($val[$sk]);=0A= =0A= if(@$flags_array[$sk] & 1)=0A= $val[$sk] =3D unserialize($val[$sk]);=0A= ------=_NextPart_000_006E_01C361C2.6B6D22D0-- From lisa@gentoo.org Wed Aug 13 22:54:46 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Wed, 13 Aug 2003 17:54:46 -0400 Subject: Compression: client or server? In-Reply-To: <007101c361e3$f2b2cb20$0b00010a@nomium.net> References: <1060801212.1753.2.camel@lisa.thedoh.com> <005001c361e1$3f9b8b00$0b00010a@nomium.net> <006101c361e2$e742be40$0b00010a@nomium.net> <007101c361e3$f2b2cb20$0b00010a@nomium.net> Message-ID: <1060811686.2043.1.camel@lisa.thedoh.com> --=-lkJr1+0SAKS9Qr6VRiKQ Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Wed, 2003-08-13 at 17:43, Justin Matlock wrote: > Ugh; sorry.. I don't know what the heck that was... Not sleeping for two > days'll do that. :) do: diff -urN oldfile newfile >patchname (sorry i like -r and -N, Brad! :-D) --=20 Regards, -Lisa --=-lkJr1+0SAKS9Qr6VRiKQ Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/OrOmy0a1Vh5Jb8URAmkaAJ9MwuxwcVNih9QpAumDOZdVciO53gCghL7R QSf11Bp6KESpRoBbwnLF6M0= =Vc8b -----END PGP SIGNATURE----- --=-lkJr1+0SAKS9Qr6VRiKQ-- From mellon@pobox.com Wed Aug 13 20:59:05 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Wed, 13 Aug 2003 22:59:05 +0300 Subject: Compression: client or server? In-Reply-To: References: <1060801212.1753.2.camel@lisa.thedoh.com> Message-ID: <20030813195905.GA29094@pobox.com> On Wed, Aug 13, 2003 at 12:53:20PM -0700, Brad Fitzpatrick wrote: > I agree. Me too. I'll mention in passing that one benefit of server-side compression you haven't mentioned is that for some methods of text compression which use dynamically built dictionaries the server is in a position to offer better compression rates, beause it sees much more data than the client. I don't this this overweighs the other arguments in favor of client-side compression though. > So, yeah: compression definitely client-side, but the server should do > stats by the client sending more data about how much it shrank it to. See > any problem with that? Only the fact that we're wasting 4 more bytes per item just for the sake of compression stats. Some people may not be very interested in stats Or do you see it as a system in which the 4 bytes are only stored for item in which clients sent compressed size, and not stored for other items? Possible, but will necessarily make the code more convoluted, and add a slight CPU penalty, probably insignificant. -- avva From brad@danga.com Thu Aug 14 01:25:33 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 13 Aug 2003 17:25:33 -0700 (PDT) Subject: Compression: client or server? In-Reply-To: <007101c361e3$f2b2cb20$0b00010a@nomium.net> References: <1060801212.1753.2.camel@lisa.thedoh.com> <005001c361e1$3f9b8b00$0b00010a@nomium.net> <006101c361e2$e742be40$0b00010a@nomium.net> <007101c361e3$f2b2cb20$0b00010a@nomium.net> Message-ID: Ryan, let me know if you want this committed. Or rather, send me an updated patch with a ChangeLog/MC_VERSION increase (MC_VERSION only if you want me to make a new release) It should also be possible to do a ->set_compression(1 | 0) on the memclient object, to selectively disable compression when storing images or other uncompressable blobs into memcache. Also, the PHP client shouldn't store the compressed version if it's larger, or not signifigantly smaller. On Wed, 13 Aug 2003, Justin Matlock wrote: > Ugh; sorry.. I don't know what the heck that was... Not sleeping for two > days'll do that. :) > > ----- Original Message ----- > From: "Brad Fitzpatrick" > To: > Sent: Wednesday, August 13, 2003 5:38 PM > Subject: Re: Compression: client or server? > > > > I only like unified diffs (-u). Please post those instead. > > > > > > On Wed, 13 Aug 2003, Justin Matlock wrote: > > > > > Doh. Corrected patch. gzipuncompress was failing in very rare cases > (null > > > on the end of the compressed data stream). > > > > > > If you don't want to patch, just remove the ",$len" from the end of the > > > gzuncompress line. :) > > > > > > ----- Original Message ----- > > > From: "Justin Matlock" > > > To: > > > Sent: Wednesday, August 13, 2003 5:24 PM > > > Subject: Re: Compression: client or server? > > > > > > > > > > You guys read my mind. :) > > > > > > > > I've been doing this for a few days now... in tests, it takes 400k > text > > > > (the linux words file), and takes it down to 100k. The extra time it > took > > > > to compress (a few milliseconds) was insignificant compared to the > speed > > > > increase from the size reduction. As far as CPU load goes -- I didn't > > > > notice a major increase in CPU, but it's somewhat hard to measure with > > > PHP. > > > > My client machine is a Linux box on a AMD Athlon 700, 256mb RAM, so > your > > > > results may vary. > > > > > > > > Attached patch to make the PHP API use gzip compression; it also > updates > > > the > > > > Documentation file about the new $options['compress'] configuration > > > > variable, which allows complete disabling of compression, or setting a > > > > minimum threshold to activate compression (I recommend 10240 from > > > testing). > > > > > > > > Justin > > > > > > > > > > > > ----- Original Message ----- > > > > From: "Brad Fitzpatrick" > > > > To: > > > > Sent: Wednesday, August 13, 2003 3:53 PM > > > > Subject: Re: Compression: client or server? > > > > > > > > > > > > > I agree. > > > > > > > > > > The issue then is stats, though. It'd be interesting to know how > much > > > > > physical vs. logical data is in memcache. Based on that, you could > > > adjust > > > > > the compression threshold on the clients. I imagine adding an > option to > > > > > the Perl MemCachedClient API that says, "Compress after this size". > > > > > > > > > > At lunch just now Evan and I realized another point to add it to the > > > > > server, though: what if somebody had more memcache servers than web > > > > > nodes? But we don't buy this argument, because in our experience > you > > > > > can't just _buy_ a memcache machine. We were unable to find a > system or > > > > > parts vendor who could get us a machine with a crap processor and > tons > > > of > > > > > RAM. > > > > > > > > > > So, yeah: compression definitely client-side, but the server should > do > > > > > stats by the client sending more data about how much it shrank it > to. > > > See > > > > > any problem with that? > > > > > > > > > > > > > > > On Wed, 13 Aug 2003, Lisa Marie Seelye wrote: > > > > > > > > > > > On Wed, 2003-08-13 at 14:44, Brad Fitzpatrick wrote: > > > > > > > Thoughts? > > > > > > > > > > > > What CPU resources are required to compress the data? How much of > a > > > lag > > > > > > will it introduce to the System? > > > > > > > > > > > > The best way to go about storing compressed data is for the client > to > > > > > > _send the data in a compressed format_ and then know well enough > to > > > > > > decompress it when they get it back. > > > > > > > > > > > > The server should store whatever the client sends them and don't > mess > > > > > > with it -- put the burden on the user, not the server. > > > > > > > > > > > > > > > > > > -- > > > > > > Regards, > > > > > > -Lisa > > > > > > > > > > > > > > > > > > > > > > > > > > > From jmat@shutdown.net Thu Aug 14 01:44:24 2003 From: jmat@shutdown.net (Justin Matlock) Date: Wed, 13 Aug 2003 20:44:24 -0400 Subject: PHP API 1.0.9 References: Message-ID: <00a301c361fd$3ba70e90$0b00010a@nomium.net> This is a multi-part message in MIME format. ------=_NextPart_000_00A0_01C361DB.B4352620 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: 7bit I ran into a few more tiny problems in the PHP API. I couldn't figure out why my server kept trying to open connections to "-1". :) It was rebuilding the server bucket every single time it tried to make a connection -- it didn't store the list of servers between calls. This, of course, wasted some CPU cycles. The hash function was returning odd values -- sometimes it would send back negative numbers, which would cause get_sock to try to assign a negative bucket position. So what was happening is about 1/2 the time, it was trying to open a socket to "", and failing. I added some more debugging lines (I pulled them back out), which showed what was happening: [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) get_sock(): hash 2.5527987333199E+15 [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) host = buckets[2.5527987333199E+15 % 4] [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) bucket chosen = $this->bucket[-2] [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) HEY! that bucket doesn't exist! This is going to fail! [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) get_sock(): calling sock_to_host with '' [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) sock_to_host(): trying host '' [03Aug13 200514] (DBUG) <10.1.0.11> (u:1) sock_to_host(): Host address was not in the format of host:port Don't ask me why the results of modulus hash was -2... I skipped the part in math class where they taught scientific notation arithmetic. :) I replaced the hash function with a simpler CRC32 (that I could understood -- plus pregs in PHP aren't all that fast), passing it through sprint("%u") to force it to always be positive. This also seems to make the host selection much more evenly spread out among the servers. I also added the minimum compression piece and the set_compression(1|0) component. Ryan; check out these changes and see if they make sense to you. And Brad... I think I've got this diff thing down. ;-) J ----- Original Message ----- From: "Brad Fitzpatrick" To: Sent: Tuesday, August 12, 2003 8:04 PM Subject: PHP API 1.0.9 > New PHP release is up, with the fix for the protocol parsing bug. > > http://www.danga.com/memcached/dist/php-memcached-1.0.9.tar.gz > > - Brad > > ------=_NextPart_000_00A0_01C361DB.B4352620 Content-Type: application/octet-stream; name="php-memcached-1.0.9c.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="php-memcached-1.0.9c.patch" diff -urN php-memcached-1.0.9/Documentation = php-memcached-1.0.9c/Documentation=0A= --- php-memcached-1.0.9/Documentation Tue Aug 12 20:09:53 2003=0A= +++ php-memcached-1.0.9c/Documentation Wed Aug 13 20:38:00 2003=0A= @@ -158,7 +158,8 @@=0A= MC_ERR_LOADITEM_BYTES // _load_items bytes read larger than bytes = available=0A= MC_ERR_GET // failed to get value associated with key=0A= =0A= -=0A= +// Turns compression on or off; 0=3Doff, 1=3Don=0A= +MemCacheClient::set_compression($setting)=0A= =0A= EXAMPLE:=0A= set_servers($options["servers"]);=0A= $this->debug =3D $options["debug"];=0A= + $this->compress =3D $options["compress"];=0A= $this->cache_sock =3D array();=0A= }=0A= =0A= @@ -550,6 +567,23 @@=0A= }=0A= =0A= =0A= + /**=0A= + * temporarily sets compression on or off=0A= + * turning it off, and then back on will result in the compression = threshold going=0A= + * back to the original setting from $options=0A= + * @param int $setting setting of compression (0=3Doff|1=3Don)=0A= + */=0A= +=0A= + function set_compression($setting=3D1) {=0A= + if ($setting !=3D 0) {=0A= + $this->comp_active =3D 1;=0A= + } else {=0A= + $this->comp_active =3D 0;=0A= + }=0A= + }=0A= +=0A= +=0A= +=0A= /*=0A= * PRIVATE FUNCTIONS=0A= */=0A= @@ -637,8 +671,6 @@=0A= */=0A= function get_sock($key)=0A= {=0A= - $buckets =3D 0;=0A= -=0A= if(!$this->active)=0A= {=0A= $this->errno =3D MC_ERR_NOT_ACTIVE;=0A= @@ -652,9 +684,9 @@=0A= =0A= $hv =3D is_array($key) ? intval($key[0]) : $this->_hashfunc($key);=0A= =0A= - if(!$buckets)=0A= + if(!$this->buckets)=0A= {=0A= - $bu =3D $buckets =3D array();=0A= + $bu =3D $this->buckets =3D array();=0A= =0A= foreach($this->servers as $v)=0A= {=0A= @@ -667,14 +699,14 @@=0A= $bu[] =3D $v;=0A= }=0A= =0A= - $buckets =3D $bu;=0A= + $this->buckets =3D $bu;=0A= }=0A= =0A= $real_key =3D is_array($key) ? $key[1] : $key;=0A= $tries =3D 0;=0A= while($tries < 20)=0A= {=0A= - $host =3D @$buckets[$hv % count($buckets)];=0A= + $host =3D @$this->buckets[$hv % count($this->buckets)];=0A= $sock =3D $this->sock_to_host($host);=0A= =0A= if(is_resource($sock))=0A= @@ -783,7 +815,6 @@=0A= return trim($retval);=0A= }=0A= =0A= -=0A= /**=0A= * sends the command to the server=0A= * Possible errors set are:=0A= @@ -837,6 +868,17 @@=0A= $flags |=3D 1;=0A= }=0A= =0A= + if (($this->compress_active) && ($this->compress > 0) && = (strlen($val) > $this->compress)) {=0A= + $this->_debug("_set(): compressing data. size in:".strlen($val));=0A= + $cval=3Dgzcompress($val);=0A= + $this->_debug("_set(): done compressing data. size = out:".strlen($cval));=0A= + if ((strlen($cval) < strlen($val)) && (strlen($val) - strlen($cval) = > 2048)){=0A= + $flags |=3D 2;=0A= + $val=3D$cval;=0A= + }=0A= + unset($cval);=0A= + }=0A= +=0A= $len =3D strlen($val);=0A= if (!is_int($exptime))=0A= $exptime =3D 0;=0A= @@ -1034,6 +1076,8 @@=0A= =0A= if(strlen($val[$sk]) !=3D $len_array[$sk])=0A= continue;=0A= + if(@$flags_array[$sk] & 2)=0A= + $val[$sk] =3D gzuncompress($val[$sk]);=0A= =0A= if(@$flags_array[$sk] & 1)=0A= $val[$sk] =3D unserialize($val[$sk]);=0A= @@ -1078,12 +1122,7 @@=0A= */=0A= function _hashfunc($num)=0A= {=0A= - $hash =3D 0;=0A= -=0A= - foreach(preg_split('//', $num, -1, PREG_SPLIT_NO_EMPTY) as $v)=0A= - {=0A= - $hash =3D $hash * 33 + ord($v);=0A= - }=0A= + $hash =3D sprintf("%u",crc32($num));=0A= =0A= return $hash;=0A= }=0A= ------=_NextPart_000_00A0_01C361DB.B4352620-- From lisa@gentoo.org Thu Aug 14 06:14:39 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Thu, 14 Aug 2003 01:14:39 -0400 Subject: PHP API 1.0.9 In-Reply-To: References: Message-ID: <1060838078.2050.8.camel@lisa.thedoh.com> --=-CvIiyuotaF7jfl/Znv5D Content-Type: multipart/mixed; boundary="=-GpzZ43lSFk1pqF5p+jGA" --=-GpzZ43lSFk1pqF5p+jGA Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Tue, 2003-08-12 at 20:04, Brad Fitzpatrick wrote: > New PHP release is up, with the fix for the protocol parsing bug. >=20 > http://www.danga.com/memcached/dist/php-memcached-1.0.9.tar.gz >=20 > - Brad Here's a stupid little cosmetic patch to fix the @version: --=20 Regards, -Lisa --=-GpzZ43lSFk1pqF5p+jGA Content-Disposition: attachment; filename=002_fix_version_in_1.0.9.patch Content-Transfer-Encoding: base64 Content-Type: text/plain; name=002_fix_version_in_1.0.9.patch; charset=iso-8859-1 LS0tIE1lbUNhY2hlZENsaWVudC5pbmMucGhwLmRpc3QJMjAwMy0wOC0xNCAwMToxMDo1Ni4wNDU4 MzUzNzYgLTA0MDANCisrKyBNZW1DYWNoZWRDbGllbnQuaW5jLnBocAkyMDAzLTA4LTE0IDAxOjEx OjM4Ljg3NTMyNDMwNCAtMDQwMA0KQEAgLTUxLDcgKzUxLDcgQEANCiAgKiBAYXV0aG9yIFJ5YW4g R2lsZmV0aGVyIDxyeWFuQGdpbGZldGhlci5jb20+DQogICogQHBhY2thZ2UgTWVtQ2FjaGVkQ2xp ZW50DQogICogQGFjY2VzcyBwdWJsaWMNCi0gKiBAdmVyc2lvbiAxLjAuNw0KKyAqIEB2ZXJzaW9u IDEuMC45DQogICovDQogY2xhc3MgTWVtQ2FjaGVkQ2xpZW50DQogew0KQEAgLTExMDEsNCArMTEw MSw0IEBADQogICAgIH0NCiB9DQogDQotPz4NClwgTm8gbmV3bGluZSBhdCBlbmQgb2YgZmlsZQ0K Kz8+DQo= --=-GpzZ43lSFk1pqF5p+jGA-- --=-CvIiyuotaF7jfl/Znv5D Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.2 (GNU/Linux) iD8DBQA/Oxq+y0a1Vh5Jb8URAofyAKCtqQAsdYFGMXtQWUbsbKhAVKEL2ACfUoY7 YBqFjzv0fQlrNIp+wn5ubUY= =BUiR -----END PGP SIGNATURE----- --=-CvIiyuotaF7jfl/Znv5D-- From martine@danga.com Fri Aug 15 05:24:18 2003 From: martine@danga.com (Evan Martin) Date: Thu, 14 Aug 2003 21:24:18 -0700 Subject: memcachedclient frontend Message-ID: <1060921458.10478.1.camel@trout> --=-C1qE7Ew4Lo0AjZurRGjX Content-Type: text/plain Content-Transfer-Encoding: 7bit -- Evan Martin martine@danga.com http://neugierig.org --=-C1qE7Ew4Lo0AjZurRGjX Content-Disposition: attachment; filename=memcachedclient Content-Type: text/x-perl; name=memcachedclient; charset=UTF-8 Content-Transfer-Encoding: 7bit #!/usr/bin/perl # vim: set ts=4 sw=4 et : use strict; use Gtk2; use MemCachedClient; use Data::Dumper; $Data::Dumper::Terse = 1; my @cmds = (); my $cmd_cur = -1; my $mc = new MemCachedClient { 'servers' => [ "127.0.0.1:11211" ] }; Gtk2->init; my $win = Gtk2::Window->new("toplevel"); $win->signal_connect(delete_event => sub { Gtk2->main_quit }); $win->set_border_width(10); my $vb = Gtk2::VBox->new(0, 5); my $display = Gtk2::TextView->new; my $buffer = $display->get_buffer; $display->set_editable(0); my $scroll = Gtk2::ScrolledWindow->new; $scroll->set_policy('automatic', 'automatic'); $scroll->set_shadow_type('in'); $scroll->add($display); $vb->pack_start($scroll, 1, 1, 0); $display->modify_font(Gtk2::Pango::FontDescription->from_string("monospace")); $buffer->create_tag("command", "foreground", "blue"); $buffer->create_tag("data"); $buffer->create_tag("error", "foreground", "red"); my $entry = Gtk2::Entry->new(); $entry->signal_connect(key_press_event => \&entry_keypress); $entry->signal_connect(activate => \&entry_activate); $vb->pack_start($entry, 0, 0, 0); $win->add($vb); $win->set_title("MemCachedClient"); $win->set_default_size(400, 500); $win->signal_connect(show => sub { $entry->grab_focus }); $win->show_all; Gtk2->main; sub display { my ($level, $text) = @_; $buffer->insert_with_tags_by_name($buffer->get_end_iter, "$text\n", $level); } sub run_command { my ($text) = @_; # if we're rerunning a history command, then # we should pull it out of its old spot in the history. splice(@cmds, $cmd_cur, 1) if $cmd_cur >= 0 and $cmds[$cmd_cur] eq $text; # and in any case, add this command to the history. unshift(@cmds, $text); $cmd_cur = -1; display('command', $text); if ($text =~ /^get\s+(\S+)$/i) { my $str = $mc->get($1); if (ref $str) { $str = Dumper($str); $str =~ s/^ //gm; } if ($str) { display('data', $str); } else { display('error', "Not found."); } } elsif ($text =~ /^set\s+(\S+)\s+(.*)$/i) { my ($key, $val) = ($1, $2); if ($mc->set($key, $val)) { display('data', "Ok."); } else { display('error', "Not found."); } } elsif ($text =~ /^delete\s+(\S+)$/i) { $mc->delete($1); display('data', "Ok."); } else { display('error', "Unknown command '$text'."); } } sub entry_keypress { my ($entry, $ev) = @_; if ($ev->keyval == $Gtk2::Gdk::Keysyms{'Up'}) { $cmd_cur++ if $cmd_cur < @cmds-1; $entry->set_text($cmds[$cmd_cur]) if $cmds[$cmd_cur]; return 1; } elsif ($ev->keyval == $Gtk2::Gdk::Keysyms{'Down'}) { $cmd_cur-- if $cmd_cur >= 0; if ($cmd_cur >= 0 and $cmds[$cmd_cur]) { $entry->set_text($cmds[$cmd_cur]); } else { $entry->set_text(''); } return 1; } return 0; } sub entry_activate { my $text = $entry->get_text; if ($text =~ /\w/) { run_command($entry->get_text); $entry->set_text(""); } } --=-C1qE7Ew4Lo0AjZurRGjX-- From martine@danga.com Fri Aug 15 05:27:05 2003 From: martine@danga.com (Evan Martin) Date: Thu, 14 Aug 2003 21:27:05 -0700 Subject: memcached frontend Message-ID: <1060921625.10478.5.camel@trout> --=-uEQ7Wtut1OZsnnJ22Htq Content-Type: text/plain Content-Transfer-Encoding: 7bit Ignore that last mail... I hit the wrong button. Once gzipped data starts showing up more often in the memcache (dear Whitaker: please send in your patch), using telnet to diagnose systems using memcached starts getting even more difficult. Attached is a GTK2/Perl (http://gtk2-perl.sf.net) client for memcached, along with a screenshot of it in use. It has a command history (up/down arrows) and will automatically use Data::Dumper on references. -- Evan Martin martine@danga.com http://neugierig.org --=-uEQ7Wtut1OZsnnJ22Htq Content-Disposition: attachment; filename=memcachedclient Content-Type: text/x-perl; name=memcachedclient; charset=UTF-8 Content-Transfer-Encoding: 7bit #!/usr/bin/perl # vim: set ts=4 sw=4 et : use strict; use Gtk2; use MemCachedClient; use Data::Dumper; $Data::Dumper::Terse = 1; my @cmds = (); my $cmd_cur = -1; my $mc = new MemCachedClient { 'servers' => [ "127.0.0.1:11211" ] }; Gtk2->init; my $win = Gtk2::Window->new("toplevel"); $win->signal_connect(delete_event => sub { Gtk2->main_quit }); $win->set_border_width(10); my $vb = Gtk2::VBox->new(0, 5); my $display = Gtk2::TextView->new; my $buffer = $display->get_buffer; $display->set_editable(0); my $scroll = Gtk2::ScrolledWindow->new; $scroll->set_policy('automatic', 'automatic'); $scroll->set_shadow_type('in'); $scroll->add($display); $vb->pack_start($scroll, 1, 1, 0); $display->modify_font(Gtk2::Pango::FontDescription->from_string("monospace")); $buffer->create_tag("command", "foreground", "blue"); $buffer->create_tag("data"); $buffer->create_tag("error", "foreground", "red"); my $entry = Gtk2::Entry->new(); $entry->signal_connect(key_press_event => \&entry_keypress); $entry->signal_connect(activate => \&entry_activate); $vb->pack_start($entry, 0, 0, 0); $win->add($vb); $win->set_title("MemCachedClient"); $win->set_default_size(400, 500); $win->signal_connect(show => sub { $entry->grab_focus }); $win->show_all; Gtk2->main; sub display { my ($level, $text) = @_; $buffer->insert_with_tags_by_name($buffer->get_end_iter, "$text\n", $level); } sub run_command { my ($text) = @_; # if we're rerunning a history command, then # we should pull it out of its old spot in the history. splice(@cmds, $cmd_cur, 1) if $cmd_cur >= 0 and $cmds[$cmd_cur] eq $text; # and in any case, add this command to the history. unshift(@cmds, $text); $cmd_cur = -1; display('command', $text); if ($text =~ /^get\s+(\S+)$/i) { my $str = $mc->get($1); if (ref $str) { $str = Dumper($str); $str =~ s/^ //gm; } if ($str) { display('data', $str); } else { display('error', "Not found."); } } elsif ($text =~ /^set\s+(\S+)\s+(.*)$/i) { my ($key, $val) = ($1, $2); if ($mc->set($key, $val)) { display('data', "Ok."); } else { display('error', "Not found."); } } elsif ($text =~ /^delete\s+(\S+)$/i) { $mc->delete($1); display('data', "Ok."); } else { display('error', "Unknown command '$text'."); } } sub entry_keypress { my ($entry, $ev) = @_; if ($ev->keyval == $Gtk2::Gdk::Keysyms{'Up'}) { $cmd_cur++ if $cmd_cur < @cmds-1; $entry->set_text($cmds[$cmd_cur]) if $cmds[$cmd_cur]; return 1; } elsif ($ev->keyval == $Gtk2::Gdk::Keysyms{'Down'}) { $cmd_cur-- if $cmd_cur >= 0; if ($cmd_cur >= 0 and $cmds[$cmd_cur]) { $entry->set_text($cmds[$cmd_cur]); } else { $entry->set_text(''); } return 1; } return 0; } sub entry_activate { my $text = $entry->get_text; if ($text =~ /\w/) { run_command($entry->get_text); $entry->set_text(""); } } --=-uEQ7Wtut1OZsnnJ22Htq Content-Disposition: attachment; filename=memcachedclient.png Content-Type: image/png; name=memcachedclient.png Content-Transfer-Encoding: base64 iVBORw0KGgoAAAANSUhEUgAAASEAAAEzCAIAAADxYSLcAAAgAElEQVR42u2de1xU1d7/PxuGGWa4 KAMMmkWAZv666FMWxa+w46VUOmR4jlmmcqqT51WpWQFHLTWtePBBBe3ipbKjldnJtOQxRbQ0TK0j leQh0dSDIahcZbjMADP7+WPjMA6bYXOVgc/75cvXZs1a67v22vuzv2uvvdf+AoSQzkQAIIoiO4KQ DqewsPCaa65R2f5oWy3Lly//7Iuf2JukK5n08G0vvfRS929nRkYGAJV9ktlsPnjwYE1NjWwBq9Ua FBR00003eXl52RI/++KnG2/9E4866Uo+++JzV5BYA1dorLKyMjMzc9KkSfaJl0eSYnn5pR9//NFk Mt1+++32MtNo/XjUiYsiimJz90qCIAiC4KSIQ4bm0lUO5fv06TNq1CiHYlar1WKxnDuXbzQaP/zw w/r6+pEjRzZ6v5oyHqoOZPe2Zx+IeadTi9vytNOWqzNixIjffvtN9qc77rjj008/VavV7u7uDgIb MGAAgBMnTmg0GpVKJcnJId3Dw6NZjTmoSxRFi8UiaczDQ63X+3l7exUUFNjndEU/lrbpsWPHL85b steW8spLkXcNvzZ6yiftrPm6Ab7THhk29GaDRq3KL6z4Ykfu3m9Pt7YSTf1O+ExpcxtsxZ00RsoT PeWTttlK2/RY+/vqqnuw5gQG4MiRI+Xl5X379tVqtXLDOgwePDg7O9vX11eSkyQwAKWlpXq9vmWN Sb5LkpbVapHw8tJFREScOnVKpfK44i7ONf1Y/0D3YP9jJ/MHAAjw9w4N7gsAxk1m1fg21zmgf583 Xh65cVPmiuVLRUvxjYMHjR7zQPquYxZhQGuramevmmvKWmxMgwnVeLTNVvv6qjtoDMC+ffvq6uoc Rnf19fVjx46trq728vJy0JjkqQYPHgxg6NChR48e9fX1DQ0NlX7atWuXxWJxdj9mm9uQxGUvMJs3 E0XRYZDqovdju3btin5w5Nv/OAm3vg89eGvGNycef+R2aXcEQXj8z7eOHTVQp/P4Kfv8yrWHjZW1 0sX787RfHxg5EBA//Gf2gP6+o0eEiqL45rs/HPpXPoDYxyO2/W/2nl3vQT0UHndmnxay12SpBKNK 6wfgvZXRfn21bm7C7/kV6zZmHfv1IgCVyu2vU2//w73XV1XXbd+V++VXuQBGjhz5p0mPD7jGp6DQ uHLt9ydOlUijfNlWCQIem3hr1P2DdFr18ZPFtoPivDFSHps7crLLK1Yf+tMfb7JvTNqmxwCkpaVJ 9bi0Q1Or1TqdTqvV2o8JJZ24u7s3vVUTBEGj0WRnZw8dOhTAsGHDbD/t3LnT09PTx8dHo9G0rLH6 +no7aV2hMatV7Ngr7tUiPT39nXfWvv+PnZW1N4y85/oX4pMljZlryh778/Camqqnn1ks1uXFTp/y 1KP9l676AYIaQHX5kb/89YPbbtG//PLfN368O/apd4fdpH/66Se+3b/HKgQNvdnw/nur64UQi8Uf lkrpIAJDJEcxbcZGdf0+tVr844NRs/8a9cQz71oF/ePT7q6rq/7LX1/29rwUGzu9vvon4LGwsLAl ietLLvw0buw9s54a/7c5H4lC3+ZaNenh/1K51/1t5hsmY27I9dempq6Q9sJ5YxoPnHGTWTXeyS6H 9b+wJPGwfWMeiHln97Zno6OjG88Bl3VoKpXKy8vL29u7qcZs91oOGlOpVL6+vkePHrUX2I4dOzw9 Pf38/Hx9fVUqVQsas1jqa2trL6ur0ZtJ/4ui1UHcLurHysrKso4W3D/65pLKsF9PlhjLTth258EH bpn9Yoqp1g0e4z7ZXr12xSiN26fwvBvAP//5T6iH/XgiGBC2fPo21EN//i2kX79+HuIJ6Ib08dEa y8+qdPerBG0zZmPqgP/djyeeMHiI/4buoTF/GDLnpWUmk2gS712+5rhKqAHw/vvvQz0E6sg9h31n PH2dWvwVuqjmWhU9fuiLc9+qqjLDc/ypC2rbXrTYGI2dQ3Oyy7KNaaji8o2cxmX9mCAI7u7u7u7u bm5uTX9qroiHh4evr69Doq+vr3R75lBQ1XSQarFY6+vr7HxX/WUf1pDSM/wYgG1pR+NmPVhSUvmP jzOsQh/b7vjrtZs/fMWuTyDW/V4r3tiQwXqN2VgudePlbQHWSnNNmbHS1LePZ8GlGsDkYCswwHvy xNtvHtLPX+/l7aURBAHWKnNNmb6vtqL0mFn4/zBJrub/XbYSCnOd2Vzi5ibAeslJqwL9dcaSn2vd IkRTDVBj2wsnjXE4cAp22bExnFK23YNJREVFZWVlybtKOT9mqatz0Jj9WNHa3OXQ5Th11myudfPy cvst93s3z/+y7U7ZJdMLzz9TXl7aeJWCYH/Vt52K9vuu0fod+7Xovj+M2ppeC0HnYCvp1agDh0+n rvpHWcmZqsqSbVs3S8XLK0w+3mpRDHLaq6KTVlVW1Xq418MzUHozTkljmvoxxbvcuNeCIHSfQ+/k MZfNzzTnl9pgyzaLuGPHDkEQoqKiAAwfPvzo0aN9+vRxcGUquRlFS11d3WVBWerr6+0nGIEecj8m tfzt9w54WH8RrabaWjdb4q49OZEj/rD7W3N1DUJD/B9/5I5X/3sn5C7eDtsbPzm4/PWJVTXpew6U WizWIYODHvvT7XNfTQPQt49n9o87zuSVefsOuXfkEFuRb787+cgjj2787He1Rvfk1LuTV+2V7VUn rfru+9M3Drnpx+NlAf4+0eNvUdKYpn5M+S5LfxYVV95zzz1ff1/eHV52tT/pZWn6zKrpLLrDlIQS W9Ikh6+vb1ZW1vDhw6UpkOzsbElmLfsxm8YsFovFUm+1WqxWq+zVwnX9mEbrd/KMCZY+gvZOjXvj Nfuz7b+NivB5Z0WM3s83O+fC7r1HNciG9r6m++uwXVgkLvrvnVP/POipv/xRrfb4veBS5nc5GvEn 6Ealrvn++VlPGwyGC0VVe/eftBX5aMvxv00N+XDdmKpqy1cZv2qEXNleddKqjZ/+OufpSUsW31R+ yfzlV9lKGtPUjynfZenPNf/4ccZfnkhICKysqp0yY2t3mIV3QtNnVhJ33nnnnXfeKVtk+PDhsq7P ZmvXrl22SQ4AtikQo9Ho6elpb6vhEbX0TnBJSckHH3wwe/as8vKyujpp2sNaX19XV1cvTXWIorh3 777+/a955JFHpPJ3R0510fcVNfU7m86G2SWK7uI5d8tvAsxWQW9xu84q9HMo1dy2IF5SWU+4ieWA RRS8rUL/ereBANzEiyrrr4JYIwpai3CtynrichGLhzXHzXpehIfF7TqL20CHtrXYKgH1Kku2m1gk QmNxC1FZf22xMbY6W7XL9n+6iRdVlhwBNSI8alVjuvLY5f7y+eHMj+zdTnV1dWlpaXV1texsu06n 0+v13t7eDmoxm81Go7Gurk62lIeHh4+Pj6enp4OLk2xZLBYfHx/bM+i6urqKigqj0eju7m6zlZGR ERsb66ixhQsXajSa2lpzfX2d5LikaXypEaIoms21Tz755Pjx420aGxr+VxDShWT/8J69xkRRrK+v N5vN9fX1zU3Qy44VpSGbE415eHg0fZdKsgXA4V0q+3TJlqSxK8aK/v7+b7/9dqv2dtLDt332xXs8 6qQrmfTwbbJ6aG090qy9g6dqca5f1paTNlzhxwghHYjkx9zYEYR0KtQYIdQYIdQYIYQaI4QaI4Qa I4RQY4RQY4RQY4QQaowQ10ClPKvtO0SEEPtPBnWYxgDMmDGDnUvIunXrOFYkxAXHij0AQYDCz0/Y 1pjb55dNJOQq+zHl3wJq51eDOuijQ40qaiok2URCOFYkxKU0lp6OoUOhViMkBOvXNyRWVGDiRGi1 CA9Hfn6jY5F8i23DiQtqmlO2Tlnryg0BWLMGAQEwGLB9uzNDyjl3DpGR0GgQGQmuJicdoLFp05CY iMpK7N2L/fsbEufNQ1AQSksRE4O4OMeRVYtDLNmcsnXKWlduCEBeHs6eRVIS5s51Zkg5cXGIiEB5 OcLDER/PM4q0e85Do8G//43QUNxwAzZsaEjcuhVZWdBq8cwzGDSoY1omW6es9VaRkACdDlOnwvYY op2Nz8jAsWPQahEXB7sYA4S01Y9t2YLDhzFuHPR6rF3bkFhUhOBguLvDzw+lpR3TMtk6Za23Cj8/ AFCrYQsT1c7Gl5XBYAAAg6HD9p30ao3ddRe2bcPvv2PrVsyb15A4YACqqmCxQBTR9DPGyufi7HPK 1ilrvQ2G7HHSeEFAk4htjol6PYqKAODiRej1PKNIuzU2aRJ++QW1tSgvh+27dDExmD8fRiPy8jB9 +hX59XocOaKoZoecsnXKWm+tIQecND4kBAcOOOZ3SBw9GikpMJmwYgVGj+YZRTpCY488Ai8vvP46 PvywITExEcXF6N8fkZGwi8YOAH//O0aOVPTwyiGnbJ2y1ltryAEnjU9MRHS0Y50OicuWITMTvr44 eBDJyTyjSJOBDxR/wzQtLY3vKxICYN26dUreCeY3TAnplmNFQgg1Rgg1Rgg11iqEjn3pnZDerDGL xTJr1qw+ffr4+fk9//zzTiLnEkLaorHU1NTc3NyTJ0+eOHHi+PHjb775JjuRkI7U2ObNm5csWWIw GAIDAxcvXrxp0yb7X5OSku644w6j0cieJUSi1e/d5+bm3nLLLdL2zTfffPz4cdtP69ev37BhQ2Zm po+PD3uWkDZqrKqqysvLS9r28vKqrKyUttPS0l577bX9+/cHBASwWwlp+1hRp9NVVVXZ9Obt7S1t x8bGTpo0KTg4mH1KSLs0duONN+bk5EjbOTk5Q4YMkbazsrL27dt3+PBh9ikh7dLY5MmTFy5cWFRU VFxc/Oqrrz766KNSemho6ObNm5977rmKigr7/Hx0Rqix1vHCCy+EhoYOHDhw4MCBgwYNmj17tu2n sLCwhQsXzpw5k91KiI1Wz3moVKrVq1evXr3aPlG8vAJ5woQJEyZMkP2JEPoxQgg1Rgg1Rgihxgih xgihxhTionFbCKEfI6SHauz8eUREQKfDokUtxFjpnnFbCOnuGps7FxERKCmB2dyY6EJxWwjpYlr9 nsfOnfj5Z2i1mDMHS5c2JLpQ3BZCursfKy5GUBAABAY2JrpQ3BZCurvG/P1x4UKDBmy4dNwWQrqX xqKikJyMmhqsWtWY6NJxWwjpXhpLSsKhQ/D3h07XQowVie4ft4WQTqXtcVvKynDDDSguZh+SXken x2158UUYjXj3XYwYwd4mpAVUbSjj74/+/TFoELZsYQcS0tH3YwBefhmVlfj55w57FEYINUYI6aqx oiAICj/RYfsilX1+2URC6MfagiiKTYUkm0gINUYI6UKNrVmzJiAgwGAwbN++XUqpqKiYOHGiVqsN Dw/Pty1EUcy5c+ciIyM1Gk1kZKSSJ3WE9HCN5eXlnT17Nikpae7cuVLKvHnzgoKCSktLY2Ji4mwL URQTFxcXERFRXl4eHh4eHx/Po0J6Em15PpaQkKDT6aZOnWp77WPr1q1ZWVlarfaZZ54Z1PoZ/YyM jGPHjmm12ri4uGHDhvGokN6uMT8/PwBqtdpisUgpRUVFwcHBoihardY2fOC+rKzMYDAAMBgMpR21 NoYQ1x0rNmXAgAFVVVUWi0WSmf1PgiDYpNhcol6vLyoqAnDx4kW9Xs+jQqgxR2JiYubPn280GvPy 8qZfubglJCTkwIEDDvkdEkePHp2SkmIymVasWDF69GgeFUKNOZKYmFhcXNy/f//IyMiRVy5uSUxM jI6OdhhAOiQuW7YsMzPT19f34MGDycnJPCqkJ9H2tS2E9Fo6fW0LIaSrx4qEEGqMEGqMkJ6Iqjs0 wjbpyDfyCf1Y2/XjhA75hnYXhH2RvqevJGdaGiIj4ekJf39Mn37Ftygd2LMHY8dCq0VAAGJjG3Iq N0SosR6F8mvBW29h/nwUF+PkSfTrh0cfbTbn8uWIi0NJCXJzce21mDy5wy46xFU11hlxW2Q5dw6R kdBoEBkJ29M7WevKw75ER2PNGgBYuxa2xxudEfYlPR3jx8PbG3o9Fi9GZmazOXfuxP33Q6eDvz8W LMChQzwne73GOiNuiyxxcYiIQHk5wsNhW+8ia1152JdVq5CSgqoqpKZi5cqW29khAQr37cOQIS3n vHQJS5fi3nt5Tvb6OY8ui9uSkYFjx6DVIi4OtvUustZlkW1SaCimTEF0NCZPRlhYV/RvTg6efNLx k8bNSfSaa3DwIM/JXu/HuixuS1kZDAYAMBga65S1LktzTZo6Fd98gylTlN6AKb9Va8qhQxg1CsuW YcyYFnKKIsrK8NRTePZZnpO9XmOdF7dFEGC/CEavbzBx8SJs611krcsaaq5Jr7+OyZPxxhvtUpQS tm3Dww9j40Y8/rii/H37Yv58fP01z8ler7HOi9sSEgL7RTCjRyMlBSYTVqyAbb2LrHVZQ7JNys7G 99/jo4/www84erTldrb5fmzlSsyahfR0PPBACzknTsS//oW6OhQXY9EicBU4NdaJcVsSExEd3Zhz 2TJkZsLXFwcPwrbeRda6rCHZJsXHIz4eKhUSEpCQ0Op2Kp+BnDMH587htttaftI1bRpmzoS3NwYP xunT+Oyz1hki3R9XjdvCqDHkKtLD47YwagxxLVwvbgujxpAefj+Gqx23hVFjSM/XGCGk0zUmCILA CS9COtWPMfwKIRwrEkKNEUKNEUKoMUKoMUKoMUIINUYINUYI6SiN8T0PQpTQxu8E8yUPQjhWJIQa I4QaI4RQY4T0bo3JTj9yTpJQY4QQaowQagzNPEnj4zVCjRFCqDFCqDFCSDfSGCf0CTXWiXz33Xf3 3HMPjwehxjqLBQsWvPbaazwepOeh6ibt+JoBJAn9GCGEGiOEGiOEGut6hMs09yuPU2f3fxt+QvOL KtoT1qeddXbDs8XtKh4/G6IoOnmDsVu93CjtTtOdauehba7CrjljOraHnR/N7lNnl2mSY0VCeqjG TCbThg0bwsPD2zyGlL0U2bbT09OHDh2qVqtDQkLWr18vJVZUVEycOFGr1YaHh+fn59uXWrZsWXBw sJeX19KlS9sw0MWVcQ9lDck2Sba4LOHh4Rs2bDCZTC02SdbQmTNnxowZ4+npOWbMmDNnzjjp4YKC goiICJ1Ot2DBghZtLViwQKfT3X333QUFBU6yyVrvpDrXrFkTEBBgMBi2b9/eKzTW1MWfPn06ISHh +uuv//zzzxctWtRJI4Rp06YlJiZWVlbu3bt3//79UuK8efOCgoJKS0tjYmLi4uLs83/77bcHDhzI z8/fvXt3i7tj3ypbI+1bK2tItkmyxWUNLVq06PPPPw8JCUlISHB+msoauv/+++fNm1daWjpu3Lgn nnjCSQ/Hx8ePGDGipKSkpqamxX6ura0tKSm57777EhISnGSTtd5Jdebl5Z09ezYpKWnu3LldPE52 ZqZAAWvXrhXbTVRUVEhIyOLFi/Pz8x1+sh1yhemyGWzb1157bVJS0rFjx8xms+3Xfv36nTt3ThTF srIyf39/+1IO7WlO8Era0Jwh2SYp3Ecb+fn5S5YsCQ0NjYqKai6PE0OiKNbU1Gi1WifWAwICCgsL pbPCecMASDkLCwsDAwOV7FFT6x1bJ4DS0lJRFM1ms7u7u9g5rF27VolkNmzYcHXGimq1WqPRqFSd +4rJli1bDh8+PG7cOL1ev3btWimxqKgoODjY3d3dz8+vtLTUPv+AAQOcK6pV1mUNyTaptahUKo1G o1arW7vvX3311U033aRSqbRarXMHVVpaajAYAEj/O0fKExgY6NCfDii33v46/fz8pNPMYrF0l3uy rvRjoiieOnUqLi4uKCgoJiZmx44dtnRBEOrr69vgx9zd3U0mkyiKFy5caJo5PT3dz89P2g4ODpZy tmiiDX7MarXa/mzOUNMmyRaXZceOHTExMf369YuLizt16pSSrrY3FBQUtHXrVrPZfOnSJYfdcfjT YDCcP39eoR+Tcp4/f95gMNj/5HA0nVjv2DplBzW9zo+FhYUlJyf/5z//eeihh+zvx0JCQg4cONCG CgcPHvzxxx9XV1cnJyfbEidNmvTLL7/U1taWl5d7enpKiTExMfPnzzcajXl5edOnT2/xVlC5H9Pr 9UeOHLH9KWtItkmyxWVZtGjRQw89dObMmeTk5LCwMCc5ZQ2ZTKbAwECz2bx48WLnhsaOHZuammoy mVJSUlrsfClnamrquHHj7NMdjqZy651U51V+dNbFfqw5PvnkEx8fH1mX4rzgl19+GRgY6O/vv27d OlvmTz/9dMiQISqV6tZbb92zZ4+UWFVVNX36dC8vr+uuu279+vUdeMFbunSpl5eXrR5ZQ7JNki3e TmQNffDBB3q93s/Pb9WqVc3NdkiJBQUFd999t1arfeWVV1r0Oa+88opWq73rrrsKCgqcHE1Z651R pxM/1oFurVV+TJAMFxYWtijFtLS0GTNmdLH+6+rqPD09u9HAmhBg3bp10dHRLWbLyMiIjY3t7s+g v/zyyxtuuIEHlbguqu7cOEEQAgMDP/jgAx4nQo111o0ijxBxdfi+IiHdT2PtWblACDXGIRwhHCsS Qo0RQqgxQqgxQqgxQgg1Rgg1Rgg1RgjpOo3xJQ9ClNOWd4L5ngchHCsSQo0RQo0RQqgxQqgxQqgx Qgg1Rgg1Rgihxgihxgihxggh1Bgh1Bgh1BghhBojhBojhBojhFBjhLgu7Yo/Jghw8tkB21c/OvvT BF1miJDu5cdEsXUnfZu/xNNaQx1rXXlxQWj41yJpaYiMhKcn/P0xfTqKiprNuWcPxo6FVouAAMTG NuRUbohwrNijUH4teOstzJ+P4mKcPIl+/fDoo83mXL4ccXEoKUFuLq69FpMnd9hFh1w1jRUUICIC Oh0WLGhMrKjAxInQahEejvx8Z8Vlc9ouuvZXX+V1ynLuHCIjodEgMhKFhQ2J5883NH7RokZDyq1H R2PNGgBYuxa2qPayxdtJejrGj4e3N/R6LF6MzMxmc+7cifvvh04Hf38sWIBDh3hKu77G4uMxYgRK SlBT05g4bx6CglBaipgYxMU5Ky6b03bdtb8AK69Tlrg4RESgvBzh4YiPb0icOxcRESgpgdks415a tL5qFVJSUFWF1FSsXOmseGvHn05y7tuHIUNaznnpEpYuxb338pR2/TmP3bvxyy/QavHSS1i+vCFx 61ZkZUGrxTPPYNAgZ8U7I6csGRk4dgxaLeLiMGxY41X/55+h1WLOHCxd2mrroaGYMgXR0Zg8GWFh XXF4cnLw5JP48ENFEr3mGhw8yFPa9f1YaSkMBgAN/0sUFSE4GO7u8PNDaamz4p2RU5ayssZ22ooX FyMoCAACA1so3pz1qVPxzTeYMkXpDZjyW7WmHDqEUaOwbBnGjGkhpyiirAxPPYVnn+Up7foaCwho mLy6eLExccAAVFXBYoEowmp1vMRaLIpyOpxAznPKXsvtDen1je3U6xsS/f1x4UKDhJyf6M1Zf/11 TJ6MN95ol6KUsG0bHn4YGzfi8ccV5e/bF/Pn4+uveUq7vsbGjkVqKkwmpKQ0JsbEYP58GI3Iy8P0 6VfkDwnBgQOKcur1OHJEUU5ZHAyNHo2UFJhMWLECo0c3JEZFITkZNTVYtcqxuBLr2dn4/nt89BF+ +AFHjzor3s77sZUrMWsW0tPxwAMt5Jw4Ef/6F+rqUFyMRYsaR8XEhTW2dCn27YNeD42mMTExEcXF 6N8fkZEYOfKK/ImJiI5uPDOc5Pz73zFypKKcsjgYWrYMmZnw9cXBg0hObkhMSsKhQ/D3h07neLIq sR4fj/h4qFRISEBCgrPizQlJ4QzknDk4dw633dbyk65p0zBzJry9MXgwTp/GZ5+1zhDpAgQAoigW 2qa3myctLW3GjBk9Y7fLynDDDSgu5glA2sK6deuibU9vmicjIyM2NrbXPYN+8UUYjXj3XYwYwVOF dAWq3rbD/v7o3x+DBmHLFh590i3vx1ydl19GZSV+/rktz9wIocYI6aEaY/RaQjpMYxaLZdasWX36 9PHz83v++eetSh4PE0KNKSc1NTU3N/fkyZMnTpw4fvz4m2++yU4kpCM1tnnz5iVLlhgMhsDAwMWL F2/atMn+16SkpDvuuMNoNLJnCZFo9dx9bm7uLbfcIm3ffPPNx48ft/20fv36DRs2ZGZm+vj4sGcJ aaPGqqqqvLy8pG0vL6/KykppOy0t7bXXXtu/f39AQAC7lZC2jxV1Ol1VVZVNb97e3tJ2bGzspEmT goOD2aeEtEtjN954Y05OjrSdk5Mz5PIa3aysrH379h0+fJh9Ski7NDZ58uSFCxcWFRUVFxe/+uqr j17+nktoaOjmzZufe+65iooK+/x8dEaosdbxwgsvhIaGDhw4cODAgYMGDZo9e7btp7CwsIULF86c OZPdSoiNVs95qFSq1atXr1692j5RvLwGeMKECRMmTJD9iRD6MUIINUYINUYIocYI6Tq6xTpoBl4h 9GMdoB8nMPAKA69QY6R11wIGXiFt15hs6BPl0ViUw8ArpJdqTDb0ifJoLMph4BXSS+c8ZEOftDPG iiwMvEJ6qR+TDX3SzhgrsjDwCumlGpMNfaI8GovzazkDrzDwCjUmH/pEeTQWJzDwCgOvUGNAM6FP lEdjcQIDrzDwSo+k7XFbrm7oEwZeIVeRTo/bcnVDnzDwCnEt2vK+4tUNfcLAK6SH34/haoc+YeAV 0vM1RghxTY299x58fDplHo1zc8SVNCYI+POfW3H6Kj+/Fy3CoUN84ZzQjwFZWThxouPbUliIy1/S J6R3a+z55/E//3NFiuxCFOVPUqUMonhFzubqbOohBQFr1iAgAAYDtm9vSCwoaFgGs2ABDzNxNY09 /TR27EBBQWOK7EIU5YtbZHPK1tkceXk4exZJSZg7tyElPh4jRqCkBDU1PMzE1TTm5YWnnsKKFY0p GRl48cWGhSi7d3dM01pVZ0ICdDpMndo4iCrYZ7kAAAHiSURBVN29Gy+8AK0WL73Ew0xcTWMAZs/G xo0oK2v4U3YhSjtpVZ1+fgCgVje+ul9a2licENfTmMGAiRPx9tsNf8ouRLEfCrYB2Trd3RuWQF+8 2ELxgIDG4oS4nsak+yWbxmQXotikonBxiwOydQ4ejI8/RnV146v4zTF2LFJTYTIhJUVmioUQF9DY oEGNX5+QXYgioXxxiwPNLW6ZOxfBwRg8uIXiS5di3z7o9dBoeJjJVaTta1sI6bV0+toWQkiXjBUJ IdQYIdQYIdQYIYQaI4QaI4QaI4RQY4RQY4QQaowQaowQaowQQo0RQo0RQo0RQqgxQqgxQqgxQgg1 Rgg1RgihxgihxgihxgghClG1Kve6devYZYR0lsaUfBiVEMKxIiHUGCHUGCGEGiOkW9CKOY+0tDT2 FyESyqcAWzd3P2bMGHYuIXv27OkUPyYRFhbGLiaE92OEuLjGBIYtJ4R+jBBqjBBqjBBCjRFCjRFC jRFCqDFCqDFCCDVGCDVGCDVGCKHGCKHGCKHGCCHUGCE9UmOiKLLvCKEfI4QaI6Sn07pv5rTqczyE EDCmBCEcKxJCjRFCqDFCqDFCqDFCCDVGCDVGCDVGCKHGCKHGCCHUGCGdjwpAYWFhRkYG+4IQQojr 8X/aY2TBdNJepAAAAABJRU5ErkJggj== --=-uEQ7Wtut1OZsnnJ22Htq-- From brad@danga.com Fri Aug 15 06:04:50 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Thu, 14 Aug 2003 22:04:50 -0700 (PDT) Subject: memcached frontend In-Reply-To: <1060921625.10478.5.camel@trout> References: <1060921625.10478.5.camel@trout> Message-ID: This will only work with Perl-stored data. If you store a compressed, serialized object from, say, PHP, then this client is useless. Maybe we should have a convention for the top 4 bits of the opaque flags about which API created it? Then a client like this could do the right thing. BTW, all: I'm off on vacation for a week, with little or no net access. Please, exchange patches/ideas, but they won't make their way into releases until I get back. - Brad On Thu, 14 Aug 2003, Evan Martin wrote: > Ignore that last mail... I hit the wrong button. > > Once gzipped data starts showing up more often in the memcache (dear > Whitaker: please send in your patch), using telnet to diagnose systems > using memcached starts getting even more difficult. > > Attached is a GTK2/Perl (http://gtk2-perl.sf.net) client for memcached, > along with a screenshot of it in use. It has a command history (up/down > arrows) and will automatically use Data::Dumper on references. > > -- > Evan Martin > martine@danga.com > http://neugierig.org > From martine@danga.com Fri Aug 15 06:45:27 2003 From: martine@danga.com (Evan Martin) Date: 14 Aug 2003 22:45:27 -0700 Subject: memcached frontend In-Reply-To: References: <1060921625.10478.5.camel@trout> Message-ID: <1060926311.584.3.camel@lulu> On Thu, 2003-08-14 at 22:04, Brad Fitzpatrick wrote: > This will only work with Perl-stored data. If you store a compressed, > serialized object from, say, PHP, then this client is useless. > > Maybe we should have a convention for the top 4 bits of the opaque flags > about which API created it? Then a client like this could do the right > thing. The point here is that the interesting data is only available to Perl anyway, because it's all been Storable'd. So yeah, only for Perl. I wouldn't want to even attempt to implement the Python pickle format in Perl, either. I suppose you could mix clients to the same memcached if you were either careful to keep their keys separate or careful to use a common storage format (ints, strings), but that's not what I'm using it for and I primarily wrote this client for my own purposes. :) -- Evan Martin martine@danga.com http://neugierig.org From brion@pobox.com Fri Aug 15 14:34:33 2003 From: brion@pobox.com (Brion Vibber) Date: Fri, 15 Aug 2003 06:34:33 -0700 Subject: Multiple delete & set? Message-ID: <3F3CE169.7000202@pobox.com> I've noticed that while 'get' supports fetching many items in one call, 'delete' and 'set' & co don't seem to. Would it be useful to have such a thing? If we need to touch potentially a few hundred items in one go (say, invalidating cached HTML of many pages due to a change of state in some resource they share), it might be nice to not have to go through a command/ack sequence for every one. On the other hand, it might not be a huge win, and such cases should be rarer than reads. -- brion vibber (brion @ pobox.com) From whitaker@danga.com Fri Aug 15 19:00:59 2003 From: whitaker@danga.com (Brad Whitaker) Date: Fri, 15 Aug 2003 11:00:59 -0700 (PDT) Subject: [patch] Compression for Perl API Message-ID: This message is in MIME format. The first part should be readable text, while the remaining parts are likely unreadable without MIME-aware tools. Send mail to mime@docserver.cac.washington.edu for more info. ---1774023665-1090092404-1060970459=:7242 Content-Type: TEXT/PLAIN; charset=US-ASCII I've had this on my todo list for some time, but after it was implemented in the PHP API, I couldn't let Perl get behind. :P A diff to MemCachedClient.pm is attached. It uses Compress::Zlib to do gzip compression on inserts and decompression on gets. There is currently a constant minimum compression gain of 20%. That is, if it does the GZIP on write and it saves less than 20%, it will insert the uncompressed version in an effort to speed up later gets. This number should probably be tweaked later (and maybe bet set by the user?), but for now this seems reasonable. A new 'compress_threshold' key can now be passed to the constructor to determine the minimum size threshold before a value is compressed. Since compression is disabled by default, this key also serves as the global on/off switch. To change its value later there is also a set_compress_threshold() method. For instances when compression should be temporarily disabled regardless of data size (such as inserting compressed images), there is a new enable_compress() method which takes a true or false value. Curious as to how compression affected speed, I wrote up a little test inserting a set of test data repeatedly. Below is the output of 3 runs showing the number of seconds get/set actions took with and without compression: compress: no sets: => 5.33751511573792 gets: => 6.15450966358185 compress: yes sets: => 5.281329870224 gets: => 8.92959928512573 compress: no sets: => 5.56796407699585 gets: => 6.13490855693817 compress: yes sets: => 5.11762177944183 gets: => 8.82273375988007 compress: no sets: => 5.89584612846375 gets: => 6.15535402297974 compress: yes sets: => 5.19342517852783 gets: => 8.98201024532318 After writing this mail, I checked and saw that Brad committed this patch just before he left. As a result, I've put the new Perl client live at test.livejournal.org and I'm watching to make sure everything continues to run smoothly. -- Brad Whitaker whitaker@danga.com ---1774023665-1090092404-1060970459=:7242 Content-Type: TEXT/plain; name="MemCachedClient.pm.diff" Content-Transfer-Encoding: BASE64 Content-ID: Content-Description: Content-Disposition: attachment; filename="MemCachedClient.pm.diff" bWFpbiAtPiB3Y210b29scyAgICAgICAgICBjZ2ktYmluL01lbUNhY2hlZENs aWVudC5wbQ0KLS0tIGN2cy93Y210b29scy9tZW1jYWNoZWQvYXBpL3Blcmwv TWVtQ2FjaGVkQ2xpZW50LnBtCTIwMDMtMDctMjggMDk6Mzg6MzIuMDAwMDAw MDAwIC0wNzAwDQorKysgY2dpLWJpbi9NZW1DYWNoZWRDbGllbnQucG0JMjAw My0wOC0xNSAwOTo1OTo0MS4wMDAwMDAwMDAgLTA3MDANCkBAIC0xMSw5ICsx MSwyMCBAQA0KIA0KIHBhY2thZ2UgTWVtQ2FjaGVkQ2xpZW50Ow0KIA0KLXVz ZSB2YXJzIHF3KCRWRVJTSU9OKTsNCisjIGZsYWcgZGVmaW5pdGlvbnMNCit1 c2UgY29uc3RhbnQgRl9TVE9SQUJMRSA9PiAxOw0KK3VzZSBjb25zdGFudCBG X0NPTVBSRVNTID0+IDI7DQorDQorIyBzaXplIHNhdmluZ3MgcmVxdWlyZWQg YmVmb3JlIHNhdmluZyBjb21wcmVzc2VkIHZhbHVlDQordXNlIGNvbnN0YW50 IENPTVBSRVNTX1NBVklOR1MgPT4gMC4yMDsgIyBwZXJjZW50DQorDQordXNl IHZhcnMgcXcoJFZFUlNJT04gJEhBVkVfWkxJQik7DQogJFZFUlNJT04gPSAi MS4wLjYiOw0KIA0KK0JFR0lOIHsNCisgICAgJEhBVkVfWkxJQiA9IGV2YWwg InVzZSBDb21wcmVzczo6WmxpYiAoKTsgMTsiOw0KK30NCisNCiBteSAlaG9z dF9kZWFkOyAgICMgaG9zdCAtPiB1bml4dGltZSBtYXJrZWQgZGVhZCB1bnRp bA0KIG15ICVjYWNoZV9zb2NrOyAgIyBob3N0IC0+IHNvY2tldA0KIA0KQEAg LTI2LDYgKzM3LDkgQEANCiAgICAgJHNlbGYtPnNldF9zZXJ2ZXJzKCRhcmdz LT57J3NlcnZlcnMnfSk7DQogICAgICRzZWxmLT57J2RlYnVnJ30gPSAkYXJn cy0+eydkZWJ1Zyd9Ow0KICAgICAkc2VsZi0+eydzdGF0cyd9ID0ge307DQor ICAgICRzZWxmLT57J2NvbXByZXNzX3RocmVzaG9sZCd9ID0gJGFyZ3MtPnsn Y29tcHJlc3NfdGhyZXNob2xkJ307DQorICAgICRzZWxmLT57J2NvbXByZXNz X2VuYWJsZSd9ICAgID0gMTsNCisNCiAgICAgcmV0dXJuICRzZWxmOw0KIH0N CiANCkBAIC00Myw2ICs1NywxNiBAQA0KICAgICAkc2VsZi0+eydkZWJ1Zyd9 ID0gJGRiZzsNCiB9DQogDQorc3ViIHNldF9jb21wcmVzc190aHJlc2hvbGQg ew0KKyAgICBteSAoJHNlbGYsICR0aHJlc2gpID0gQF87DQorICAgICRzZWxm LT57J2NvbXByZXNzX3RocmVzaG9sZCd9ID0gJHRocmVzaDsNCit9DQorDQor c3ViIGVuYWJsZV9jb21wcmVzcyB7DQorICAgIG15ICgkc2VsZiwgJGVuYWJs ZSkgPSBAXzsNCisgICAgJHNlbGYtPnsnY29tcHJlc3NfZW5hYmxlJ30gPSAk ZW5hYmxlOw0KK30NCisNCiBzdWIgZm9yZ2V0X2RlYWRfaG9zdHMgew0KICAg ICAlaG9zdF9kZWFkID0gKCk7DQogfQ0KQEAgLTEzMSwxNSArMTU1LDM0IEBA DQogICAgIHJldHVybiAwIHVubGVzcyAkc2VsZi0+eydhY3RpdmUnfTsNCiAg ICAgbXkgJHNvY2sgPSAkc2VsZi0+Z2V0X3NvY2soJGtleSk7DQogICAgIHJl dHVybiAwIHVubGVzcyAkc29jazsNCisNCisgICAgdXNlIGJ5dGVzOyAjIHJl dHVybiBieXRlcyBmcm9tIGxlbmd0aCgpDQorDQogICAgICRzZWxmLT57J3N0 YXRzJ30tPnskY21kbmFtZX0rKzsNCiAgICAgbXkgJGZsYWdzID0gMDsNCiAg ICAgJGtleSA9IHJlZiAka2V5IGVxICJBUlJBWSIgPyAka2V5LT5bMV0gOiAk a2V5Ow0KLSAgICBteSAkcmF3X3ZhbCA9ICR2YWw7DQorICAgIG15ICRyYXdf dmFsID0gJHNlbGYtPnsnZGVidWcnfSA/ICR2YWwgOiB1bmRlZjsNCiAgICAg aWYgKHJlZiAkdmFsKSB7DQogICAgICAgICAkdmFsID0gU3RvcmFibGU6OmZy ZWV6ZSgkdmFsKTsNCi0gICAgICAgICRmbGFncyB8PSAxOw0KKyAgICAgICAg JGZsYWdzIHw9IEZfU1RPUkFCTEU7DQogICAgIH0NCisNCiAgICAgbXkgJGxl biA9IGxlbmd0aCgkdmFsKTsNCisNCisgICAgaWYgKCRIQVZFX1pMSUIgJiYg JHNlbGYtPnsnY29tcHJlc3NfdGhyZXNob2xkJ30gJiYgJHNlbGYtPnsnY29t cHJlc3NfZW5hYmxlJ30gJiYNCisgICAgICAgICRsZW4gPj0gJHNlbGYtPnsn Y29tcHJlc3NfdGhyZXNob2xkJ30pIHsNCisNCisgICAgICAgIG15ICRjX3Zh bCA9IENvbXByZXNzOjpabGliOjptZW1HemlwKCR2YWwpOw0KKyAgICAgICAg bXkgJGNfbGVuID0gbGVuZ3RoKCRjX3ZhbCk7DQorDQorICAgICAgICAjIGRv IHdlIHdhbnQgdG8ga2VlcCBpdD8NCisgICAgICAgIGlmICgkY19sZW4gPCAk bGVuKigxIC0gQ09NUFJFU1NfU0FWSU5HUykpIHsNCisgICAgICAgICAgICAk dmFsID0gJGNfdmFsOw0KKyAgICAgICAgICAgICRsZW4gPSAkY19sZW47DQor ICAgICAgICAgICAgJGZsYWdzIHw9IEZfQ09NUFJFU1M7DQorICAgICAgICB9 DQorICAgIH0NCisNCiAgICAgJGV4cHRpbWUgPSBpbnQoJGV4cHRpbWUgfHwg MCk7DQogICAgIG15ICRjbWQgPSAiJGNtZG5hbWUgJGtleSAkZmxhZ3MgJGV4 cHRpbWUgJGxlblxyXG4kdmFsXHJcbiI7DQogICAgICRzb2NrLT5wcmludCgk Y21kKTsNCkBAIC0yMjYsNiArMjY5LDggQEANCiAgICAgbXkgJHNvY2sgPSBz aGlmdDsNCiAgICAgbXkgJG91dHJlZiA9IHNoaWZ0Ow0KIA0KKyAgICB1c2Ug Ynl0ZXM7ICMgcmV0dXJuIGJ5dGVzIGZyb20gbGVuZ3RoKCkNCisNCiAgICAg bXkgJWZsYWdzOw0KICAgICBteSAldmFsOw0KICAgICBteSAlbGVuOyAgICMg a2V5IC0+IGludGVuZGVkIGxlbmd0aA0KQEAgLTI1OCw3ICszMDMsOCBAQA0K ICAgICAgICAgICAgIGZvcmVhY2ggKGtleXMgJWxlbikgew0KICAgICAgICAg ICAgICAgICBuZXh0IHVubGVzcyBleGlzdHMgJHZhbHskX307DQogICAgICAg ICAgICAgICAgIG5leHQgdW5sZXNzIGxlbmd0aCgkdmFseyRffSkgPT0gJGxl bnskX307DQotICAgICAgICAgICAgICAgICR2YWx7JF99ID0gU3RvcmFibGU6 OnRoYXcoJHZhbHskX30pIGlmICRmbGFnc3skX30gJiAxOw0KKyAgICAgICAg ICAgICAgICAkdmFseyRffSA9IENvbXByZXNzOjpabGliOjptZW1HdW56aXAo JHZhbHskX30pIGlmICRIQVZFX1pMSUIgJiYgJGZsYWdzeyRffSAmIEZfQ09N UFJFU1M7DQorICAgICAgICAgICAgICAgICR2YWx7JF99ID0gU3RvcmFibGU6 OnRoYXcoJHZhbHskX30pIGlmICRmbGFnc3skX30gJiBGX1NUT1JBQkxFOw0K ICAgICAgICAgICAgICAgICAkb3V0cmVmLT57JF99ID0gJHZhbHskX307DQog ICAgICAgICAgICAgfQ0KICAgICAgICAgICAgIHJldHVybiAxOw0KQEAgLTI5 Miw4ICszMzgsMTEgQEANCiAgICAgJ3NlcnZlcnMnID0+IFsgIjEwLjAuMC4x NToxMTIxMSIsICIxMC4wLjAuMTU6MTEyMTIiLCANCiAgICAgICAgICAgICAg ICAgICAgIjEwLjAuMC4xNzoxMTIxMSIsIFsgIjEwLjAuMC4xNzoxMTIxMSIs IDMgXSBdLA0KICAgICAnZGVidWcnID0+IDAsDQorICAgICdjb21wcmVzc190 aHJlc2hvbGQnID0+IDEwXzAwMCwNCiAgIH07DQogICAkbWVtYy0+c2V0X3Nl cnZlcnMoJGFycmF5X3JlZik7DQorICAkbWVtYy0+c2V0X2NvbXByZXNzX3Ro cmVzaG9sZCgxMF8wMDApOw0KKyAgJG1lbWMtPmVuYWJsZV9jb21wcmVzcygw KTsNCiANCiAgICRtZW1jLT5zZXQoIm15X2tleSIsICJTb21lIHZhbHVlIik7 DQogICAkbWVtYy0+c2V0KCJvYmplY3Rfa2V5IiwgeyAnY29tcGxleCcgPT4g WyAib2JqZWN0IiwgMiwgNCBdfSk7DQpAQCAtMzI4LDYgKzM3NywxMCBAQA0K IGFzIHBvc3NpYmxlLCBhcyB0aGlzIG1vZHVsZSBjdXJyZW50bHkgYWxsb2Nh dGVzIG1lbW9yeSBmb3IgYnVja2V0DQogZGlzdHJpYnV0aW9uIHByb3BvcnRp b25hbCB0byB0aGUgdG90YWwgaG9zdCB3ZWlnaHRzLg0KIA0KK1VzZSBDPGNv bXByZXNzX3RocmVzaG9sZD4gdG8gc2V0IGEgY29tcHJlc3Npb24gdGhyZXNo b2xkLCBpbiBieXRlcy4NCitWYWx1ZXMgbGFyZ2VyIHRoYW4gdGhpcyB0aHJl c2hvbGQgd2lsbCBiZSBjb21wcmVzc2VkIGJ5IEM8c2V0PiBhbmQNCitkZWNv bXByZXNzZWQgYnkgQzxnZXQ+Lg0KKw0KIFRoZSBvdGhlciB1c2VmdWwga2V5 IGlzIEM8ZGVidWc+LCB3aGljaCB3aGVuIHNldCB0byB0cnVlIHdpbGwgcHJv ZHVjZQ0KIGRpYWdub3N0aWNzIG9uIFNUREVSUi4NCiANCkBAIC0zNDMsNiAr Mzk2LDE5IEBADQogVGhlIGZvcm1hdCBpcyBhbiBhcnJheXJlZiBvZiBpZGVu dGljYWwgZm9ybSBhcyBkZXNjcmliZWQgaW4gdGhlIEM8bmV3Pg0KIGNvbnN0 cnVjdG9yLg0KIA0KKz1pdGVtIEM8c2V0X2RlYnVnPg0KKw0KK1NldHMgdGhl IEM8ZGVidWc+IGZsYWcuICBTZWUgQzxuZXc+IGNvbnN0cnVjdG9yIGZvciBt b3JlIGluZm9ybWF0aW9uLg0KKw0KKz1pdGVtIEM8c2V0X2NvbXByZXNzX3Ro cmVzaG9sZD4NCisNCitTZXRzIHRoZSBjb21wcmVzc2lvbiB0aHJlc2hvbGQu IFNlZSBDPG5ldz4gY29uc3RydWN0b3IgZm9yIG1vcmUgaW5mb3JtYXRpb24u DQorDQorPWl0ZW0gQzxlbmFibGVfY29tcHJlc3M+DQorDQorVGVtcG9yYXJp bHkgZW5hYmxlIG9yIGRpc2FibGUgY29tcHJlc3Npb24uICBIYXMgbm8gZWZm ZWN0IGlmIEM8Y29tcHJlc3NfdGhyZXNob2xkPg0KK2lzbid0IHNldCwgYnV0 IGhhcyBhbiBvdmVycmlkaW5nIGVmZmVjdCBpZiBpdCBpcy4NCisNCiA9aXRl bSBDPGdldD4NCiANCiBteSAkdmFsID0gJG1lbS0+Z2V0KCRrZXkpOw0K ---1774023665-1090092404-1060970459=:7242-- From jmat@shutdown.net Fri Aug 15 19:46:50 2003 From: jmat@shutdown.net (Justin Matlock) Date: Fri, 15 Aug 2003 14:46:50 -0400 Subject: [patch] Compression for Perl API References: Message-ID: <001001c3635d$9698cad0$0b00010a@nomium.net> So what those results say, is it takes 2 seconds longer to retrieve compressed data? I'm very surprised at the big increase in get times with compression. I'm not seeing anything near those numbers on the PHP side. How big was your test data set, and what kind of data did it consist of? (I'd like to run the same test on my code and see what comes out). Something else you might want to consider for the Perl client: I now do a quick check of the first 10 or so characters of the data to be compressed, and abort the compression if it turns out the data is a JPEG, PNG, or GIF. You just have to look for '\x89PNG' for PNG, '0xffd8' for JPG, and 'GIF' for GIF files, starting from position 0. I found that one of my not-so-bright-about-compression semi-developer (he should just stick to HTML) was trying to store 3k JPEG files in the memcache and couldn't figure out why compression was taking so long. As soon as I get full power back here, I'm going to run some similar benchmarks on the PHP client, also on the PHP5 client, which uses streams, and the C/Zend API I've written that will actually compile into the PHP binary. So far, the C program is just a little faster than the pre-compiled PHP code (using the Zend Optimizer pre-compiler) leading me to believe it's going to be better to leave it as plain PHP... I just hope my development box comes back when Con-Ed gets around to plugging us back in. I had stolen it's UPS for one of my servers last week, and hadn't gotten around to replacing it yet (doh!). It blew my 403 day uptime... dangit. :) J ----- Original Message ----- From: "Brad Whitaker" To: Sent: Friday, August 15, 2003 2:00 PM Subject: [patch] Compression for Perl API > I've had this on my todo list for some time, but after it was implemented > in the PHP API, I couldn't let Perl get behind. :P > > A diff to MemCachedClient.pm is attached. It uses Compress::Zlib to do > gzip compression on inserts and decompression on gets. There is currently > a constant minimum compression gain of 20%. That is, if it does the GZIP > on write and it saves less than 20%, it will insert the uncompressed > version in an effort to speed up later gets. This number should probably > be tweaked later (and maybe bet set by the user?), but for now this seems > reasonable. > > A new 'compress_threshold' key can now be passed to the constructor to > determine the minimum size threshold before a value is compressed. > Since compression is disabled by default, this key also serves as the > global on/off switch. To change its value later there is also a > set_compress_threshold() method. > > For instances when compression should be temporarily disabled regardless > of data size (such as inserting compressed images), there is a new > enable_compress() method which takes a true or false value. > > Curious as to how compression affected speed, I wrote up a little test > inserting a set of test data repeatedly. Below is the output of 3 runs > showing the number of seconds get/set actions took with and without > compression: > > compress: no > sets: => 5.33751511573792 > gets: => 6.15450966358185 > compress: yes > sets: => 5.281329870224 > gets: => 8.92959928512573 > > compress: no > sets: => 5.56796407699585 > gets: => 6.13490855693817 > compress: yes > sets: => 5.11762177944183 > gets: => 8.82273375988007 > > compress: no > sets: => 5.89584612846375 > gets: => 6.15535402297974 > compress: yes > sets: => 5.19342517852783 > gets: => 8.98201024532318 > > After writing this mail, I checked and saw that Brad committed this patch > just before he left. As a result, I've put the new Perl client live at > test.livejournal.org and I'm watching to make sure everything continues to > run smoothly. > > -- > Brad Whitaker > whitaker@danga.com From martine@danga.com Fri Aug 15 20:05:14 2003 From: martine@danga.com (Evan Martin) Date: Fri, 15 Aug 2003 12:05:14 -0700 Subject: minor configure change Message-ID: <1060974314.20087.20.camel@trout> --=-nNTRKqoDYLU/HdFPq+mb Content-Type: text/plain Content-Transfer-Encoding: 7bit This should fix: ./configure: line 2895: test: =: unary operator expected -- Evan Martin martine@danga.com http://neugierig.org --=-nNTRKqoDYLU/HdFPq+mb Content-Description: Content-Disposition: attachment; filename=mc-conf.patch Content-Type: text/x-patch; charset=UTF-8 Content-Transfer-Encoding: 7bit Index: configure.ac =================================================================== RCS file: /home/cvspub/wcmtools/memcached/configure.ac,v retrieving revision 1.18 diff -u -r1.18 configure.ac --- configure.ac 13 Aug 2003 01:32:02 -0000 1.18 +++ configure.ac 15 Aug 2003 19:04:24 -0000 @@ -10,7 +10,7 @@ AC_ARG_WITH(libevent, AC_HELP_STRING([--with-libevent=DIRECTORY],[base directory for libevent])) -if test ${with_libevent+set} = set && test $with_libevent != no; then +if test "${with_libevent}" != "no"; then CFLAGS="$CFLAGS -I$with_libevent/include" LDFLAGS="$LDFLAGS -L$with_libevent/lib" fi --=-nNTRKqoDYLU/HdFPq+mb-- From martine@danga.com Fri Aug 15 20:10:02 2003 From: martine@danga.com (Evan Martin) Date: Fri, 15 Aug 2003 12:10:02 -0700 Subject: libevent debs Message-ID: <1060974601.20087.23.camel@trout> Debian unstable is still at 0.6. This is 0.7a: http://neugierig.org/software/libevent -- Evan Martin martine@danga.com http://neugierig.org From whitaker@danga.com Fri Aug 15 20:35:57 2003 From: whitaker@danga.com (Brad Whitaker) Date: Fri, 15 Aug 2003 12:35:57 -0700 (PDT) Subject: [patch] Compression for Perl API In-Reply-To: <001001c3635d$9698cad0$0b00010a@nomium.net> References: <001001c3635d$9698cad0$0b00010a@nomium.net> Message-ID: My data set was 40 rows, roughly 200k. The entire data set was inserted something like 20 times for each test (I forget exactly, I could run it again). The extra 2 seconds isn't so bad when you figure that it's run over many iterations. In this case 800. The benchmarks were a quick hack before I left for the day, and my computer isn't very fast, so I'll likely play with them later to make the numbers a bit more useful. I'm about to leave for the weekend, so I'll have to fully respond to this later. Here are some quick compression ration stats from our test machine: 1093 => 657 (0.40%) 5808 => 1785 (0.69%) 1799 => 933 (0.48%) 8835 => 2091 (0.76%) 1232 => 733 (0.41%) 2783 => 1352 (0.51%) 1360 => 739 (0.46%) 1349 => 771 (0.43%) 1925 => 988 (0.49%) 2035 => 1090 (0.46%) 1051 => 616 (0.41%) 1706 => 966 (0.43%) 2911 => 1489 (0.49%) 1314 => 751 (0.43%) 3423 => 1680 (0.51%) 1477 => 638 (0.57%) 3693 => 952 (0.74%) 4391 => 2247 (0.49%) 4671 => 2186 (0.53%) 5096 => 2144 (0.58%) Pretty huge savings. I'm wondering if the 20% minimum savings constant should be moved up closer to 30%? -- Brad Whitaker whitaker@danga.com On Fri, 15 Aug 2003, Justin Matlock wrote: > So what those results say, is it takes 2 seconds longer to retrieve > compressed data? I'm very surprised at the big increase in get times with > compression. I'm not seeing anything near those numbers on the PHP side. > > How big was your test data set, and what kind of data did it consist of? > (I'd like to run the same test on my code and see what comes out). > > Something else you might want to consider for the Perl client: I now do a > quick check of the first 10 or so characters of the data to be compressed, > and abort the compression if it turns out the data is a JPEG, PNG, or GIF. > You just have to look for '\x89PNG' for PNG, '0xffd8' for JPG, and 'GIF' for > GIF files, starting from position 0. I found that one of my > not-so-bright-about-compression semi-developer (he should just stick to > HTML) was trying to store 3k JPEG files in the memcache and couldn't figure > out why compression was taking so long. > > As soon as I get full power back here, I'm going to run some similar > benchmarks on the PHP client, also on the PHP5 client, which uses streams, > and the C/Zend API I've written that will actually compile into the PHP > binary. So far, the C program is just a little faster than the pre-compiled > PHP code (using the Zend Optimizer pre-compiler) leading me to believe it's > going to be better to leave it as plain PHP... > > I just hope my development box comes back when Con-Ed gets around to > plugging us back in. I had stolen it's UPS for one of my servers last week, > and hadn't gotten around to replacing it yet (doh!). It blew my 403 day > uptime... dangit. :) > > J > > ----- Original Message ----- > From: "Brad Whitaker" > To: > Sent: Friday, August 15, 2003 2:00 PM > Subject: [patch] Compression for Perl API > > > > I've had this on my todo list for some time, but after it was implemented > > in the PHP API, I couldn't let Perl get behind. :P > > > > A diff to MemCachedClient.pm is attached. It uses Compress::Zlib to do > > gzip compression on inserts and decompression on gets. There is currently > > a constant minimum compression gain of 20%. That is, if it does the GZIP > > on write and it saves less than 20%, it will insert the uncompressed > > version in an effort to speed up later gets. This number should probably > > be tweaked later (and maybe bet set by the user?), but for now this seems > > reasonable. > > > > A new 'compress_threshold' key can now be passed to the constructor to > > determine the minimum size threshold before a value is compressed. > > Since compression is disabled by default, this key also serves as the > > global on/off switch. To change its value later there is also a > > set_compress_threshold() method. > > > > For instances when compression should be temporarily disabled regardless > > of data size (such as inserting compressed images), there is a new > > enable_compress() method which takes a true or false value. > > > > Curious as to how compression affected speed, I wrote up a little test > > inserting a set of test data repeatedly. Below is the output of 3 runs > > showing the number of seconds get/set actions took with and without > > compression: > > > > compress: no > > sets: => 5.33751511573792 > > gets: => 6.15450966358185 > > compress: yes > > sets: => 5.281329870224 > > gets: => 8.92959928512573 > > > > compress: no > > sets: => 5.56796407699585 > > gets: => 6.13490855693817 > > compress: yes > > sets: => 5.11762177944183 > > gets: => 8.82273375988007 > > > > compress: no > > sets: => 5.89584612846375 > > gets: => 6.15535402297974 > > compress: yes > > sets: => 5.19342517852783 > > gets: => 8.98201024532318 > > > > After writing this mail, I checked and saw that Brad committed this patch > > just before he left. As a result, I've put the new Perl client live at > > test.livejournal.org and I'm watching to make sure everything continues to > > run smoothly. > > > > -- > > Brad Whitaker > > whitaker@danga.com > From jmat@shutdown.net Fri Aug 15 22:00:55 2003 From: jmat@shutdown.net (Justin Matlock) Date: Fri, 15 Aug 2003 17:00:55 -0400 Subject: [patch] Compression for Perl API References: <001001c3635d$9698cad0$0b00010a@nomium.net> Message-ID: <003201c36370$51f985a0$0b00010a@nomium.net> Have a good weekend -- just some notes from me before I get the hell out of NYC for the next 3 days.. Our power's gone out three moire times in the past hour, and I'm tired of it.... :) Here's what I just got back for the PHP API; the memcache server is running on a Celeron 600mhz with 512mb RAM (which also serves as my load balancer, so it's semi-loaded). The machine running the test is a AMD Athlon 1800. I'm a little confused as to how you did your test - you did 20 loads of 200k, or 20 loads of 40x200k? Anyway, here are my stats -- the data set ended up being around 450k (50,000 random words -- I'm assuming that's mostly what you're storing @ LJ). Setting with compressed data takes quite a bit longer, but getting with compressed data is actually faster... it hits that threshhold of network speeds vs. decompression speeds. I have a second run under the first; 5000 words (45k). .. Generating random data using 50000 words from /usr/share/dict/words. Generated data is 450,454 bytes in size. Starting test for UNCOMPRESSED data... doing 20 loops... Test completed in 1.927 seconds. Average SET without compression: 0.04581275 Average GET without compression: 0.05032785 Maximum Set: 0.0594160000 Maximum Get: 0.0597590000 Size of datablock sent/received: 450454 bytes With compression OFF, for SET, we get about: 9,602 kB per second With compression OFF, for GET, we get about: 8,741 kB per second Starting test for COMPRESSED data... doing 20 loops... Test completed in 4.242 seconds. Average SET with compression: 0.16641615 Average GET with compression: 0.04549155 Maximum Set: 0.1825650000 Maximum Get: 0.0505000000 Size of datablock sent/received: 206821 bytes With compression ON, for SET, we get about: 2,643 kB per second With compression ON, for GET, we get about: 9,670 kB per second Compressed data is 54 % smaller than uncompressed. ............. and now just 5000 words (45k)... it's almost identical... but you're saving 51% of your memory in the cache server. Generating data using 5000 words from wordlist... Generated data is 44,780 bytes in size. Starting test for UNCOMPRESSED data... doing 20 loops... Test completed in 0.308 seconds. Average SET without compression: 0.00834685 Average GET without compression: 0.00688475 Maximum Set: 0.0235120000 Maximum Get: 0.0109610000 Size of datablock sent/received: 44780 bytes With compression OFF, for SET, we get about: 5,239 kB per second With compression OFF, for GET, we get about: 6,352 kB per second Turning Compression ON... Starting test for COMPRESSED data... doing 20 loops... Test completed in 0.478 seconds. Average SET with compression: 0.0170215 Average GET with compression: 0.0066843 Maximum Set: 0.0217830000 Maximum Get: 0.0088190000 Size of datablock sent/received: 21853 bytes With compression ON, for SET, we get about: 2,569 kB per second With compression ON, for GET, we get about: 6,542 kB per second Compressed data is 51 % smaller than uncompressed. ...... just for entertainment purposes (I hope you're not storing this much).... Generating data using 500000 words from wordlist... Generated data is 4,507,165 bytes in size. Starting test for UNCOMPRESSED data... doing 20 loops... Test completed in 9.720 seconds. Average SET without compression: 0.47382015 Average GET without compression: 0.0120007 Maximum Set: 0.4974120000 Maximum Get: 0.0165600000 Size of datablock sent/received: 4507165 bytes With compression OFF, for SET, we get about: 9,289 kB per second With compression OFF, for GET, we get about: 366,773 kB per second Turning Compression ON... Starting test for COMPRESSED data... doing 20 loops... Test completed in 33.313 seconds. Average SET with compression: 1.65426465 Average GET with compression: 0.01120345 Maximum Set: 1.7040400000 Maximum Get: 0.0135460000 Size of datablock sent/received: 2055760 bytes With compression ON, for SET, we get about: 2,661 kB per second With compression ON, for GET, we get about: 392,873 kB per second Compressed data is 54 % smaller than uncompressed. ----- Original Message ----- From: "Brad Whitaker" To: "Justin Matlock" Cc: Sent: Friday, August 15, 2003 3:35 PM Subject: Re: [patch] Compression for Perl API > My data set was 40 rows, roughly 200k. The entire data set was inserted > something like 20 times for each test (I forget exactly, I could run it > again). The extra 2 seconds isn't so bad when you figure that it's run > over many iterations. In this case 800. > > The benchmarks were a quick hack before I left for the day, and my > computer isn't very fast, so I'll likely play with them later to make the > numbers a bit more useful. > > I'm about to leave for the weekend, so I'll have to fully respond to this > later. > > Here are some quick compression ration stats from our test machine: > > 1093 => 657 (0.40%) > 5808 => 1785 (0.69%) > 1799 => 933 (0.48%) > 8835 => 2091 (0.76%) > 1232 => 733 (0.41%) > 2783 => 1352 (0.51%) > 1360 => 739 (0.46%) > 1349 => 771 (0.43%) > 1925 => 988 (0.49%) > 2035 => 1090 (0.46%) > 1051 => 616 (0.41%) > 1706 => 966 (0.43%) > 2911 => 1489 (0.49%) > 1314 => 751 (0.43%) > 3423 => 1680 (0.51%) > 1477 => 638 (0.57%) > 3693 => 952 (0.74%) > 4391 => 2247 (0.49%) > 4671 => 2186 (0.53%) > 5096 => 2144 (0.58%) > > Pretty huge savings. I'm wondering if the 20% minimum savings constant > should be moved up closer to 30%? > > -- > Brad Whitaker > whitaker@danga.com > > On Fri, 15 Aug 2003, Justin Matlock wrote: > > > So what those results say, is it takes 2 seconds longer to retrieve > > compressed data? I'm very surprised at the big increase in get times with > > compression. I'm not seeing anything near those numbers on the PHP side. > > > > How big was your test data set, and what kind of data did it consist of? > > (I'd like to run the same test on my code and see what comes out). > > > > Something else you might want to consider for the Perl client: I now do a > > quick check of the first 10 or so characters of the data to be compressed, > > and abort the compression if it turns out the data is a JPEG, PNG, or GIF. > > You just have to look for '\x89PNG' for PNG, '0xffd8' for JPG, and 'GIF' for > > GIF files, starting from position 0. I found that one of my > > not-so-bright-about-compression semi-developer (he should just stick to > > HTML) was trying to store 3k JPEG files in the memcache and couldn't figure > > out why compression was taking so long. > > > > As soon as I get full power back here, I'm going to run some similar > > benchmarks on the PHP client, also on the PHP5 client, which uses streams, > > and the C/Zend API I've written that will actually compile into the PHP > > binary. So far, the C program is just a little faster than the pre-compiled > > PHP code (using the Zend Optimizer pre-compiler) leading me to believe it's > > going to be better to leave it as plain PHP... > > > > I just hope my development box comes back when Con-Ed gets around to > > plugging us back in. I had stolen it's UPS for one of my servers last week, > > and hadn't gotten around to replacing it yet (doh!). It blew my 403 day > > uptime... dangit. :) > > > > J > > > > ----- Original Message ----- > > From: "Brad Whitaker" > > To: > > Sent: Friday, August 15, 2003 2:00 PM > > Subject: [patch] Compression for Perl API > > > > > > > I've had this on my todo list for some time, but after it was implemented > > > in the PHP API, I couldn't let Perl get behind. :P > > > > > > A diff to MemCachedClient.pm is attached. It uses Compress::Zlib to do > > > gzip compression on inserts and decompression on gets. There is currently > > > a constant minimum compression gain of 20%. That is, if it does the GZIP > > > on write and it saves less than 20%, it will insert the uncompressed > > > version in an effort to speed up later gets. This number should probably > > > be tweaked later (and maybe bet set by the user?), but for now this seems > > > reasonable. > > > > > > A new 'compress_threshold' key can now be passed to the constructor to > > > determine the minimum size threshold before a value is compressed. > > > Since compression is disabled by default, this key also serves as the > > > global on/off switch. To change its value later there is also a > > > set_compress_threshold() method. > > > > > > For instances when compression should be temporarily disabled regardless > > > of data size (such as inserting compressed images), there is a new > > > enable_compress() method which takes a true or false value. > > > > > > Curious as to how compression affected speed, I wrote up a little test > > > inserting a set of test data repeatedly. Below is the output of 3 runs > > > showing the number of seconds get/set actions took with and without > > > compression: > > > > > > compress: no > > > sets: => 5.33751511573792 > > > gets: => 6.15450966358185 > > > compress: yes > > > sets: => 5.281329870224 > > > gets: => 8.92959928512573 > > > > > > compress: no > > > sets: => 5.56796407699585 > > > gets: => 6.13490855693817 > > > compress: yes > > > sets: => 5.11762177944183 > > > gets: => 8.82273375988007 > > > > > > compress: no > > > sets: => 5.89584612846375 > > > gets: => 6.15535402297974 > > > compress: yes > > > sets: => 5.19342517852783 > > > gets: => 8.98201024532318 > > > > > > After writing this mail, I checked and saw that Brad committed this patch > > > just before he left. As a result, I've put the new Perl client live at > > > test.livejournal.org and I'm watching to make sure everything continues to > > > run smoothly. > > > > > > -- > > > Brad Whitaker > > > whitaker@danga.com > > > From sean@chittenden.org Thu Aug 21 20:00:53 2003 From: sean@chittenden.org (Sean Chittenden) Date: Thu, 21 Aug 2003 12:00:53 -0700 Subject: [PATCH] memcached on FreeBSD... Message-ID: <20030821190053.GD4807@perrin.int.nxad.com> Howdy. When are you guys planning on releasing 1.1.9? Many of the patches used by the port have already been applied or worked around. I couldn't get CVS to build though. What version of autofuck are you guys depending on? I couldn't get it to build with 213, 253, or 257 (need to copy in install.sh or install-sh too, btw). % autoconf257 configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE If this token and others are legitimate, please use m4_pattern_allow. See the Autoconf documentation. configure.ac:6: error: possibly undefined macro: AM_CONFIG_HEADER % cp /usr/src/crypto/openssl/util/install.sh . % ./configure --with-libevent=/usr/local/lib --prefix=/usr/local configure: error: cannot run /usr/local/bin/bash ./config.sub For 1.1.8, I had to gut the -k option since that's a Linux-ism (though I noticed the #ifdef's around the mlockall() calls in CVS). Has any thought been given to using mmap(2) instead malloc(3)? OSes are generally much better about actually free()ing data back to the OS if it's been mmap(2)'ed instead of using sbrk(2) (what most malloc(3) calls use internally). If you decide to mmap(2) non-anonymous files you could get a persistent cache almost for free by mmap(2)'ing actual files (of the hash ID of the objects?) and then use MAP_SHARED | MAP_NOCORE | MAP_NOSYNC. thttpd has a nifty mmap(2) cache that may be worth peaking at if this is of interest (thttpd/mmc.[ch]). Caching aside, mmap(2) should have more portable locking semantics via mlock(2) and can be friendlier on the VM if madvise(2) is used too. mmap(2) and friends are from the BSD 4.4 spec and are likely POSIX or SUSv3 calls at this point. With Apache making heavy use of these calls, I can't imagine an OS not supporting them in this day and age. Just a design thought. -sc -- Sean Chittenden From mellon@pobox.com Thu Aug 21 20:51:43 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Thu, 21 Aug 2003 22:51:43 +0300 Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <20030821190053.GD4807@perrin.int.nxad.com> References: <20030821190053.GD4807@perrin.int.nxad.com> Message-ID: <20030821195143.GA8572@pobox.com> On Thu, Aug 21, 2003 at 12:00:53PM -0700, Sean Chittenden wrote: > Howdy. When are you guys planning on releasing 1.1.9? Hopefully in a few days. I'm finishing the sloppy queues feature (deleting expired items every 5 minutes or so instead of them taking up space in memory until they're requested or pushed off), and with Brad returning from his vacation soon, we'll intergrate it and call it 1.1.9 (that was the plan anyway). > What version of autofuck are you > guys depending on? autofuck is good. I'm not sure, I'll try build from CVS and see. > For 1.1.8, I had to gut the -k option since that's a Linux-ism (though > I noticed the #ifdef's around the mlockall() calls in CVS). Has any > thought been given to using mmap(2) instead malloc(3)? OSes are > generally much better about actually free()ing data back to the OS if > it's been mmap(2)'ed instead of using sbrk(2) (what most malloc(3) > calls use internally). We're only using malloc() to allocate chunks so large that it uses mmap(), not sbrk() (with a very few exceptions like read/write buffers and connection structures, and we're reusing those so there's no real fragmentation going on). We also never free() those large chunks, so again, no fragmentation. The fact that malloc() uses mmap() rather than sbrk() to allocate large stuff is true in Linux (for chunks>64kb IIRC... we malloc() chunks of 1Mb anyway, presently), but I haven't verified it in FreeBSD. Is it true in FreeBSD? If not, we might want to explicitly mmap() there, although this might turn out to be not really important, since we don't free and don't fragment. > If you decide to mmap(2) non-anonymous files > you could get a persistent cache almost for free by mmap(2)'ing actual > files (of the hash ID of the objects?) and then use MAP_SHARED | > MAP_NOCORE | MAP_NOSYNC. Will anyone need a persistent memcache useful? I'm not certain about that. -- avva@livejournal.com From sean@chittenden.org Thu Aug 21 21:20:39 2003 From: sean@chittenden.org (Sean Chittenden) Date: Thu, 21 Aug 2003 13:20:39 -0700 Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <20030821195143.GA8572@pobox.com> References: <20030821190053.GD4807@perrin.int.nxad.com> <20030821195143.GA8572@pobox.com> Message-ID: <20030821202039.GE4807@perrin.int.nxad.com> > > Howdy. When are you guys planning on releasing 1.1.9? > > Hopefully in a few days. I'm finishing the sloppy queues feature > (deleting expired items every 5 minutes or so instead of them taking > up space in memory until they're requested or pushed off), and with > Brad returning from his vacation soon, we'll intergrate it and call > it 1.1.9 (that was the plan anyway). Cool. > > What version of autofuck are you > > guys depending on? > > autofuck is good. I'm not sure, I'll try build from CVS and see. *gets shivers* I've spent too much time dorking with autoconf and friends and dealing with the changing semantics between versions to have anything nice to say about it, sorry. :) > > For 1.1.8, I had to gut the -k option since that's a Linux-ism (though > > I noticed the #ifdef's around the mlockall() calls in CVS). Has any > > thought been given to using mmap(2) instead malloc(3)? OSes are > > generally much better about actually free()ing data back to the OS if > > it's been mmap(2)'ed instead of using sbrk(2) (what most malloc(3) > > calls use internally). > > We're only using malloc() to allocate chunks so large that it uses > mmap(), not sbrk() (with a very few exceptions like read/write > buffers and connection structures, and we're reusing those so > there's no real fragmentation going on). We also never free() those > large chunks, so again, no fragmentation. Ah, ok... I thought that it'd eventually free itself if there was unused slabs. Setting the H option in FreeBSD's _malloc_options will internally call madvise(MADV_FREE) when free()'ing large hunks. It's an extra syscall, but it's better than swapping. > The fact that malloc() uses mmap() rather than sbrk() to allocate > large stuff is true in Linux (for chunks>64kb IIRC... we malloc() > chunks of 1Mb anyway, presently), but I haven't verified it in > FreeBSD. Is it true in FreeBSD? If not, we might want to explicitly > mmap() there, although this might turn out to be not really > important, since we don't free and don't fragment. FreeBSD uses brk(), regardless of the size. It mmap()'s the administrative structures for malloc'ed data, but that's it. If bits are never free()'ed, this is a pretty mute point and I'll go back to lurking. > > If you decide to mmap(2) non-anonymous files you could get a > > persistent cache almost for free by mmap(2)'ing actual files (of > > the hash ID of the objects?) and then use MAP_SHARED | MAP_NOCORE > > | MAP_NOSYNC. > > Will anyone need a persistent memcache useful? I'm not certain about > that. No, but by explicitly mmap()'ing non-anonymous regions, it's not overly hard to do which could be nice for keeping a cache populated across restarts, however, I have no use or desire that feature. It was more of an anecdotal bit from my webserver/proxy server writing days and would be low hanging fruit that'd fall out of using mmap() instead of relying on Linux's malloc.h bits. :) -sc -- Sean Chittenden From martine@danga.com Thu Aug 21 22:09:52 2003 From: martine@danga.com (Evan Martin) Date: Thu, 21 Aug 2003 14:09:52 -0700 Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <20030821190053.GD4807@perrin.int.nxad.com> References: <20030821190053.GD4807@perrin.int.nxad.com> Message-ID: <1061500192.8128.89.camel@trout> On Thu, 2003-08-21 at 12:00, Sean Chittenden wrote: > I couldn't get CVS to build though. What version of autofuck are you > guys depending on? I couldn't get it to build with 213, 253, or 257 > (need to copy in install.sh or install-sh too, btw). > > % autoconf257 > configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE > If this token and others are legitimate, please use m4_pattern_allow. > See the Autoconf documentation. > configure.ac:6: error: possibly undefined macro: AM_CONFIG_HEADER What version of automake are you using? Are you using the same version of aclocal? Also, could you give the output of aclocal --print-ac-dir? I hate auto* as much as the next person, but I think I'm the one who gets to try to make it work. :) > % cp /usr/src/crypto/openssl/util/install.sh . > % ./configure --with-libevent=/usr/local/lib --prefix=/usr/local > configure: error: cannot run /usr/local/bin/bash ./config.sub Try "automake --add-missing". I suppose that's not in the autogen script, but it ought to be. -- Evan Martin martine@danga.com http://neugierig.org From martine@danga.com Thu Aug 21 22:22:26 2003 From: martine@danga.com (Evan Martin) Date: Thu, 21 Aug 2003 14:22:26 -0700 Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <20030821202039.GE4807@perrin.int.nxad.com> References: <20030821190053.GD4807@perrin.int.nxad.com> <20030821195143.GA8572@pobox.com> <20030821202039.GE4807@perrin.int.nxad.com> Message-ID: <1061500945.8128.103.camel@trout> On Thu, 2003-08-21 at 13:20, Sean Chittenden wrote: > > > If you decide to mmap(2) non-anonymous files you could get a > > > persistent cache almost for free by mmap(2)'ing actual files (of > > > the hash ID of the objects?) and then use MAP_SHARED | MAP_NOCORE > > > | MAP_NOSYNC. > > > > Will anyone need a persistent memcache useful? I'm not certain about > > that. > > No, but by explicitly mmap()'ing non-anonymous regions, it's not > overly hard to do which could be nice for keeping a cache populated > across restarts, however, I have no use or desire that feature. It > was more of an anecdotal bit from my webserver/proxy server writing > days and would be low hanging fruit that'd fall out of using mmap() > instead of relying on Linux's malloc.h bits. :) This reminds me more and more of http://www.prevayler.org each day. (It's pretty hard to find any real text on that site, though.) -- Evan Martin martine@danga.com http://neugierig.org From sean@chittenden.org Thu Aug 21 22:24:39 2003 From: sean@chittenden.org (Sean Chittenden) Date: Thu, 21 Aug 2003 14:24:39 -0700 Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <1061500192.8128.89.camel@trout> References: <20030821190053.GD4807@perrin.int.nxad.com> <1061500192.8128.89.camel@trout> Message-ID: <20030821212439.GF4807@perrin.int.nxad.com> > > I couldn't get CVS to build though. What version of autofuck are you > > guys depending on? I couldn't get it to build with 213, 253, or 257 > > (need to copy in install.sh or install-sh too, btw). > > > > % autoconf257 > > configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE > > If this token and others are legitimate, please use m4_pattern_allow. > > See the Autoconf documentation. > > configure.ac:6: error: possibly undefined macro: AM_CONFIG_HEADER > > What version of automake are you using? What version do you want me to try using? I've got 1.4.5, 1.5, and 1.7.5, none seemed successful. > Are you using the same version of aclocal? Yup. > Also, could you give the output of aclocal --print-ac-dir? Depending on the version of aclocal: /usr/local/share/aclocal /usr/local/share/aclocal14 /usr/local/share/aclocal17 > I hate auto* as much as the next person, but I think I'm the one who > gets to try to make it work. :) Doh! I don't know if you've looked around at all, but pmk looks promising and vastly less hellish. http://premk.sourceforge.net/ > > % cp /usr/src/crypto/openssl/util/install.sh . > > % ./configure --with-libevent=/usr/local/lib --prefix=/usr/local > > configure: error: cannot run /usr/local/bin/bash ./config.sub > > Try "automake --add-missing". I suppose that's not in the autogen > script, but it ought to be. Duh, thanks, that helps some, but now I'm getting a syntax error in configure: % ./configure --with-libevent=/usr/local/lib --prefix=/usr/local checking build system type... i386-unknown-freebsd5.1 checking host system type... i386-unknown-freebsd5.1 checking target system type... i386-unknown-freebsd5.1 ./configure: line 1318: syntax error near unexpected token `memcached,' ./configure: line 1318: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' or with autoconf257: checking build system type... i386-unknown-freebsd5.1 checking host system type... i386-unknown-freebsd5.1 checking target system type... i386-unknown-freebsd5.1 ./configure: line 1393: syntax error near unexpected token `memcached,' ./configure: line 1393: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' autoconf can't find the macro AM_INIT_AUTOMAKE. :-/ configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE If this token and others are legitimate, please use m4_pattern_allow. See the Autoconf documentation. -sc -- Sean Chittenden From sean@chittenden.org Thu Aug 21 22:27:34 2003 From: sean@chittenden.org (Sean Chittenden) Date: Thu, 21 Aug 2003 14:27:34 -0700 Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <1061500945.8128.103.camel@trout> References: <20030821190053.GD4807@perrin.int.nxad.com> <20030821195143.GA8572@pobox.com> <20030821202039.GE4807@perrin.int.nxad.com> <1061500945.8128.103.camel@trout> Message-ID: <20030821212734.GG4807@perrin.int.nxad.com> > This reminds me more and more of http://www.prevayler.org each day. > (It's pretty hard to find any real text on that site, though.) :) Something that could write to disk would rock as an object store. This is OT, but ever play with Madeline? http://madeleine.sourceforge.net/ Neither prevayler or madeleine write their bits to disk outside of a giant snapshot, which kinda sucks if you have more bits in memory than you have RAM as it can't page anything to disk. -sc -- Sean Chittenden From martine@danga.com Thu Aug 21 22:42:06 2003 From: martine@danga.com (Evan Martin) Date: Thu, 21 Aug 2003 14:42:06 -0700 Subject: autoconf (was [PATCH] memcached on FreeBSD...) In-Reply-To: <20030821212439.GF4807@perrin.int.nxad.com> References: <20030821190053.GD4807@perrin.int.nxad.com> <1061500192.8128.89.camel@trout> <20030821212439.GF4807@perrin.int.nxad.com> Message-ID: <1061502126.7163.121.camel@trout> On Thu, 2003-08-21 at 14:24, Sean Chittenden wrote: > > > > configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE > > > If this token and others are legitimate, please use m4_pattern_allow. > > > See the Autoconf documentation. > > > configure.ac:6: error: possibly undefined macro: AM_CONFIG_HEADER > > > > What version of automake are you using? > > What version do you want me to try using? I've got 1.4.5, 1.5, and > 1.7.5, none seemed successful. It's working for me with both 1.4 and 1.7. (Each different x.y version of automake is slightly incompatible with the others... but they did change the version numbers, after all!) autoconf, on the other hand, I think has only two main families: the 2.1x series and the 2.5x+ series. Do you get any warnings with this sequence of commands? aclocal-1.4; autoheader; automake-1.4; autoconf also try it with -1.7. (Note that you should be using the same versions of autoheader and autoconf, as well.) You might also want to check things out from CVS again to make sure you don't have leftover files from somewhere messing things up. > http://premk.sourceforge.net/ The autotools have a lot of inertia, especially because they support platforms we've never even seen... > > Try "automake --add-missing". I suppose that's not in the autogen > > script, but it ought to be. > > Duh, thanks, that helps some, but now I'm getting a syntax error in > configure: Hm, wait. The autogen script *does* include --add-missing. Are you not running autogen for some reason? > ./configure: line 1318: syntax error near unexpected token `memcached,' > ./configure: line 1318: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' > > or with autoconf257: > > ./configure: line 1393: syntax error near unexpected token `memcached,' > ./configure: line 1393: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' > > autoconf can't find the macro AM_INIT_AUTOMAKE. :-/ > > configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE > If this token and others are legitimate, please use m4_pattern_allow. > See the Autoconf documentation. Because m4 is a macro language, the errors are often very misleading. Even though configure is failing, it could be any tool in the chain. If AM_INIT_AUTOMAKE is undefined, it's likely something isn't providing it, and I'd suspect automake over autoconf. (See http://www.advogato.org/person/raph/diary.html?start=349 for a discussion of this problem.) It's possible you don't even need that line (the AM_INIT_AUTOMAKE one in configure.ac); I believe you normally include it to fake compatibility with older automakes. If the above suggestions don't help, could you deleting that line and running the process again? I'd be able to diagnose this better if I could reproduce it, but as I said, it works for me on the versions I've tried... -- Evan Martin martine@danga.com http://neugierig.org From martine@danga.com Thu Aug 21 22:46:02 2003 From: martine@danga.com (Evan Martin) Date: Thu, 21 Aug 2003 14:46:02 -0700 Subject: madeleine (was Re: [PATCH] memcached on FreeBSD...) In-Reply-To: <20030821212734.GG4807@perrin.int.nxad.com> References: <20030821190053.GD4807@perrin.int.nxad.com> <20030821195143.GA8572@pobox.com> <20030821202039.GE4807@perrin.int.nxad.com> <1061500945.8128.103.camel@trout> <20030821212734.GG4807@perrin.int.nxad.com> Message-ID: <1061502362.7163.124.camel@trout> On Thu, 2003-08-21 at 14:27, Sean Chittenden wrote: > > This reminds me more and more of http://www.prevayler.org each day. > > (It's pretty hard to find any real text on that site, though.) > > :) Something that could write to disk would rock as an object store. > This is OT, but ever play with Madeline? > > http://madeleine.sourceforge.net/ That looks interesting. I keep intending to write a Ruby client for memcached, but as far as I can tell nobody but me really uses Ruby. :) -- Evan Martin martine@danga.com http://neugierig.org From sean@chittenden.org Thu Aug 21 23:12:18 2003 From: sean@chittenden.org (Sean Chittenden) Date: Thu, 21 Aug 2003 15:12:18 -0700 Subject: autoconf (was [PATCH] memcached on FreeBSD...) In-Reply-To: <1061502126.7163.121.camel@trout> References: <20030821190053.GD4807@perrin.int.nxad.com> <1061500192.8128.89.camel@trout> <20030821212439.GF4807@perrin.int.nxad.com> <1061502126.7163.121.camel@trout> Message-ID: <20030821221218.GI4807@perrin.int.nxad.com> > > http://premk.sourceforge.net/ > > The autotools have a lot of inertia, especially because they support > platforms we've never even seen... Bah, by that argument, use Win32. > > > Try "automake --add-missing". I suppose that's not in the autogen > > > script, but it ought to be. > > > > Duh, thanks, that helps some, but now I'm getting a syntax error in > > configure: > > Hm, wait. The autogen script *does* include --add-missing. Are you not > running autogen for some reason? ... oh, hey, that's handy (modifies it for BSD's naming convention). I wasn't running autoheader. *sigh* > > ./configure: line 1318: syntax error near unexpected token `memcached,' > > ./configure: line 1318: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' > > > > or with autoconf257: > > > > ./configure: line 1393: syntax error near unexpected token `memcached,' > > ./configure: line 1393: `AM_INIT_AUTOMAKE(memcached, 1.1.9-snapshot)' > > > > autoconf can't find the macro AM_INIT_AUTOMAKE. :-/ > > > > configure.ac:5: error: possibly undefined macro: AM_INIT_AUTOMAKE > > If this token and others are legitimate, please use m4_pattern_allow. > > See the Autoconf documentation. > > Because m4 is a macro language, the errors are often very > misleading. Even though configure is failing, it could be any tool > in the chain. If AM_INIT_AUTOMAKE is undefined, it's likely > something isn't providing it, and I'd suspect automake over > autoconf. *nods* 'tis be why I like pmk: less obtuse msgs. Thanks for the help. -sc -- Sean Chittenden From jdc@uwo.ca Wed Aug 20 04:01:41 2003 From: jdc@uwo.ca (Dan Christensen) Date: Tue, 19 Aug 2003 23:01:41 -0400 Subject: C API? Message-ID: <87smnxowve.fsf@uwo.ca> Just heard about memcached, and it could be perfect for several projects I'm working on. It happens that these projects are written in C, and don't involve anything to do with the web---memcached seems general enough to solve lots of different problems. So I'm just wondering if anyone is working on a C API. Thanks for creating such a great program! Dan PS: Please cc me on replies. From vibber@aludra.usc.edu Thu Aug 21 20:40:15 2003 From: vibber@aludra.usc.edu (Brion Vibber) Date: Thu, 21 Aug 2003 12:40:15 -0700 (PDT) Subject: [PATCH] memcached on FreeBSD... In-Reply-To: <20030821190053.GD4807@perrin.int.nxad.com> Message-ID: On Thu, 21 Aug 2003, Sean Chittenden wrote: > Howdy. When are you guys planning on releasing 1.1.9? Many of the > patches used by the port have already been applied or worked around. > I couldn't get CVS to build though. What version of autofuck are you > guys depending on? I couldn't get it to build with 213, 253, or 257 > (need to copy in install.sh or install-sh too, btw). The version in CVS did compile on FreeBSD 5.1 last I tried it... did it break again? Sigh... -- brion vibber (brion @ pobox.com) From brad@danga.com Fri Aug 22 01:27:48 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Thu, 21 Aug 2003 17:27:48 -0700 (PDT) Subject: C API? In-Reply-To: <87smnxowve.fsf@uwo.ca> References: <87smnxowve.fsf@uwo.ca> Message-ID: Avva was working on one (or at least, I told him Danga would fund it if he wanted to), but I haven't heard anything. Of course, if you want to make one and contribute it, we'll maintain/support it. - Brad On Tue, 19 Aug 2003, Dan Christensen wrote: > Just heard about memcached, and it could be perfect for several > projects I'm working on. It happens that these projects are written > in C, and don't involve anything to do with the web---memcached seems > general enough to solve lots of different problems. So I'm just > wondering if anyone is working on a C API. > > Thanks for creating such a great program! > > Dan > > PS: Please cc me on replies. > > From mellon@pobox.com Fri Aug 22 01:29:52 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Fri, 22 Aug 2003 03:29:52 +0300 Subject: C API? In-Reply-To: References: <87smnxowve.fsf@uwo.ca> Message-ID: <20030822002952.GA10285@pobox.com> On Thu, Aug 21, 2003 at 05:27:48PM -0700, Brad Fitzpatrick wrote: > Avva was working on one (or at least, I told him Danga would fund it if he > wanted to), but I haven't heard anything. > > Of course, if you want to make one and contribute it, we'll > maintain/support it. I started to work on one, but haven't got far. I intend to continue soon as I get sloppy calendar queues done, but of course anyone should feel free to write one if they want. -- avva@livejournal.com From martine@danga.com Fri Aug 22 02:04:23 2003 From: martine@danga.com (Evan Martin) Date: Thu, 21 Aug 2003 18:04:23 -0700 Subject: C API? In-Reply-To: <20030822002952.GA10285@pobox.com> References: <87smnxowve.fsf@uwo.ca> <20030822002952.GA10285@pobox.com> Message-ID: <1061514263.12763.24.camel@trout> On Thu, 2003-08-21 at 17:29, Anatoly Vorobey wrote: > I started to work on one, but haven't got far. I intend to continue soon > as I get sloppy calendar queues done, but of course anyone should feel > free to write one if they want. I was just talking to Whitaker about this. Maybe I'll do it tonight... -- Evan Martin martine@danga.com http://neugierig.org From mellon@pobox.com Fri Aug 22 02:18:07 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Fri, 22 Aug 2003 04:18:07 +0300 Subject: C API? In-Reply-To: <1061514263.12763.24.camel@trout> References: <87smnxowve.fsf@uwo.ca> <20030822002952.GA10285@pobox.com> <1061514263.12763.24.camel@trout> Message-ID: <20030822011806.GA13398@pobox.com> On Thu, Aug 21, 2003 at 06:04:23PM -0700, Evan Martin wrote: > On Thu, 2003-08-21 at 17:29, Anatoly Vorobey wrote: > > I started to work on one, but haven't got far. I intend to continue soon > > as I get sloppy calendar queues done, but of course anyone should feel > > free to write one if they want. > > I was just talking to Whitaker about this. Maybe I'll do it tonight... One thing worth remembering, if you do, is that multi-get should be tighter than it is in existing APIs, if you're going to the trouble of writing it in C: after splitting the gets and sending them to various servers, it's probably best not to read replies server-after-server, like perl/PHP APIs do, but use select/poll/whatever to read them simultaneously, so as not to let a slower server become a bottleneck. -- avva From mellon@pobox.com Fri Aug 22 02:19:29 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Fri, 22 Aug 2003 04:19:29 +0300 Subject: epoll, spinning Message-ID: <20030822011929.GA13342@pobox.com> I noticed there was a new epoll patch at http://www.xmailserver.org/linux-patches/nio-improve.html, as of yesterday (the patch is http://www.xmailserver.org/linux-patches/epoll-lt-2.4.21-0.17.diff ). I asked the maintainer about it and he wrote back saying it didn't contain anything exciting, and is mostly about ia32 simulation on ia64 CPUs. So there's no real reason to upgrade to it. We're still trying to track down a bug which sometimes (rarely) causes memcached processes to start eating a lot of CPUs, but continue to serve requests and not crash. Runtime stats don't help much with it, and I haven't been able to reliably reproduce it yet, so after a lot of auditing I'm willing to suspect an obscure bug in libevent or epoll. It could, of course, be memcached code nevertheless. If anyone outside LiveJournal running memcached experiences this, please let me know your configuration/general circumstances/etc. As Brad wrote a while ago, it's completely safe to just kill and restart the offending server if this happens to it; I hope we'll have a real fix soon. -- avva From martine@danga.com Fri Aug 22 02:27:47 2003 From: martine@danga.com (Evan Martin) Date: Thu, 21 Aug 2003 18:27:47 -0700 Subject: C API? In-Reply-To: <20030822011806.GA13398@pobox.com> References: <87smnxowve.fsf@uwo.ca> <20030822002952.GA10285@pobox.com> <1061514263.12763.24.camel@trout> <20030822011806.GA13398@pobox.com> Message-ID: <1061515667.12758.30.camel@trout> On Thu, 2003-08-21 at 18:18, Anatoly Vorobey wrote: > > I was just talking to Whitaker about this. Maybe I'll do it tonight... > > One thing worth remembering, if you do, is that multi-get should be > tighter than it is in existing APIs, if you're going to the trouble > of writing it in C: after splitting the gets and sending them to various > servers, it's probably best not to read replies server-after-server, > like perl/PHP APIs do, but use select/poll/whatever to read them > simultaneously, so as not to let a slower server become a bottleneck. I was worrying about this, too... are there cases where you'd want to be able to integrate the memcache client reads into a centralized event loop? In that case, we need to somehow (either by returning file descriptors or requiring a register_fd callback argument) let the user do the select(). Same for put(). Yikes. And then, should we really use select? Or is this another job for libevent? :\ -- Evan Martin martine@danga.com http://neugierig.org From mellon@pobox.com Fri Aug 22 02:46:40 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Fri, 22 Aug 2003 04:46:40 +0300 Subject: C API? In-Reply-To: <1061515667.12758.30.camel@trout> References: <87smnxowve.fsf@uwo.ca> <20030822002952.GA10285@pobox.com> <1061514263.12763.24.camel@trout> <20030822011806.GA13398@pobox.com> <1061515667.12758.30.camel@trout> Message-ID: <20030822014640.GA13683@pobox.com> On Thu, Aug 21, 2003 at 06:27:47PM -0700, Evan Martin wrote: > I was worrying about this, too... are there cases where you'd want to be > able to integrate the memcache client reads into a centralized event > loop? In that case, we need to somehow (either by returning file > descriptors or requiring a register_fd callback argument) let the user > do the select(). Same for put(). Yikes. Hmm. Not sure. I'd guess we can get off the ground with set/retrieve operations being atomic for the client, and then see if there's any demand for doing it the other way. > And then, should we really use select? Or is this another job for > libevent? :\ Dunno about libevent, but epoll is probably an overkill here (at least demanding it is). Of course, we could rely on libevent and let it use select/poll if it doesn't find epoll. Or we could eliminate the dependency and just do select/poll ourselves. What's could the maximum number of servers per request, in a large installation that uses memcache now? 50-60? I don't think the benefits of epoll kick in yet, at such small numbers. -- avva@livejournal.com From brad@danga.com Fri Aug 22 02:49:03 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Thu, 21 Aug 2003 18:49:03 -0700 (PDT) Subject: C API? In-Reply-To: <20030822014640.GA13683@pobox.com> References: <87smnxowve.fsf@uwo.ca> <20030822002952.GA10285@pobox.com> <1061514263.12763.24.camel@trout> <20030822011806.GA13398@pobox.com> <1061515667.12758.30.camel@trout> <20030822014640.GA13683@pobox.com> Message-ID: Agreed. Just send all requests and select/poll on the 1-50 sockets. On Fri, 22 Aug 2003, Anatoly Vorobey wrote: > On Thu, Aug 21, 2003 at 06:27:47PM -0700, Evan Martin wrote: > > I was worrying about this, too... are there cases where you'd want to be > > able to integrate the memcache client reads into a centralized event > > loop? In that case, we need to somehow (either by returning file > > descriptors or requiring a register_fd callback argument) let the user > > do the select(). Same for put(). Yikes. > > Hmm. Not sure. I'd guess we can get off the ground with set/retrieve > operations being atomic for the client, and then see if there's any > demand for doing it the other way. > > > And then, should we really use select? Or is this another job for > > libevent? :\ > > Dunno about libevent, but epoll is probably an overkill here (at least > demanding it is). Of course, we could rely on libevent and let it use > select/poll if it doesn't find epoll. Or we could eliminate the > dependency and just do select/poll ourselves. What's could the maximum > number of servers per request, in a large installation that uses > memcache now? 50-60? I don't think the benefits of epoll kick in yet, > at such small numbers. > > -- > avva@livejournal.com > > From martine@danga.com Fri Aug 22 06:36:23 2003 From: martine@danga.com (Evan Martin) Date: 21 Aug 2003 22:36:23 -0700 Subject: first stab at a C API Message-ID: <1061530572.25046.9.camel@lulu> --=-gFraGkq9rYuzecL4carf Content-Type: text/plain Content-Transfer-Encoding: 7bit This is super-basic: only connects to localhost, only has get and set. The plan is this: - buffer reads (readline() should do a full read and scan for newlines) - don't buffer writes (the kernel buffers it anyway / TCP_CORK) I'm not sure how the functions should look. I'll attach my header. I figure that eventually, we'd like something like get() to: - support printf-style formatting: get("user:%d", userid) - support hash keys: should these be ints or char* and len? and maybe: - support some sort of serialization/deserialization callbacks(?) but that produces a Windowsean function with 10 arguments. Also, I took a shot at allowing weighting without using an array of buckets. Instead, we use the linked list of servers, but that means this is O(n) * number of time we retry to find a live server. If anyone wants to comment on my C code, I'd appreciate it. You can read it at http://bunny.darktech.org/svn/memcache-c , and you can check it out with this subversion command: svn checkout http://bunny.darktech.org/svn/memcache-c mydestdir -- Evan Martin martine@danga.com http://neugierig.org --=-gFraGkq9rYuzecL4carf Content-Disposition: attachment; filename=memcache.h Content-Type: text/x-c-header; name=memcache.h; charset=UTF-8 Content-Transfer-Encoding: 7bit #ifndef MEMCACHE_H #define MEMCACHE_H #include /* time_t */ typedef struct _MemCacheClient MemCacheClient; MemCacheClient* memcachec_new(void); void* memcachec_get(MemCacheClient *mc, char *key, int *rlen); int memcachec_set(MemCacheClient *mc, char *key, void *val, int len, time_t exp); #endif /* MEMCACHE_H */ --=-gFraGkq9rYuzecL4carf-- From brad@danga.com Mon Aug 25 17:57:18 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 25 Aug 2003 09:57:18 -0700 (PDT) Subject: Multiple delete & set? In-Reply-To: <3F3CE169.7000202@pobox.com> References: <3F3CE169.7000202@pobox.com> Message-ID: We could always add a command like commands_start and commands_end to cork/uncork replies from the server's end, but there wouldn't be a big win. Perhaps in the future, if we're feeling sick. On Fri, 15 Aug 2003, Brion Vibber wrote: > I've noticed that while 'get' supports fetching many items in one call, > 'delete' and 'set' & co don't seem to. > > Would it be useful to have such a thing? If we need to touch potentially > a few hundred items in one go (say, invalidating cached HTML of many > pages due to a change of state in some resource they share), it might be > nice to not have to go through a command/ack sequence for every one. On > the other hand, it might not be a huge win, and such cases should be > rarer than reads. > > -- brion vibber (brion @ pobox.com) > > From brad@danga.com Mon Aug 25 18:08:12 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 25 Aug 2003 10:08:12 -0700 (PDT) Subject: libevent debs In-Reply-To: <1060974601.20087.23.camel@trout> References: <1060974601.20087.23.camel@trout> Message-ID: Have you nudged the Debian libevent maintainer? On Fri, 15 Aug 2003, Evan Martin wrote: > Debian unstable is still at 0.6. This is 0.7a: > > http://neugierig.org/software/libevent > > -- > Evan Martin > martine@danga.com > http://neugierig.org > > From martine@danga.com Mon Aug 25 19:33:45 2003 From: martine@danga.com (Evan Martin) Date: Mon, 25 Aug 2003 11:33:45 -0700 Subject: libevent debs In-Reply-To: References: <1060974601.20087.23.camel@trout> Message-ID: <1061836425.2684.5.camel@trout> On Mon, 2003-08-25 at 10:08, Brad Fitzpatrick wrote: > Have you nudged the Debian libevent maintainer? http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=197044 libevent0: New version of libevent is available from upstream (75 days old) And I think my debs don't have epoll support. As I recall, you had to do some header trickery? If you'd send me more information I can update them, or send you the stuff you need to build the debs yourself (it's trivial). -- Evan Martin martine@danga.com http://neugierig.org From mellon@pobox.com Mon Aug 25 19:41:02 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Mon, 25 Aug 2003 21:41:02 +0300 Subject: libevent debs In-Reply-To: <1061836425.2684.5.camel@trout> References: <1060974601.20087.23.camel@trout> <1061836425.2684.5.camel@trout> Message-ID: <20030825184102.GA15197@pobox.com> On Mon, Aug 25, 2003 at 11:33:45AM -0700, Evan Martin wrote: > On Mon, 2003-08-25 at 10:08, Brad Fitzpatrick wrote: > > Have you nudged the Debian libevent maintainer? > > http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=197044 > libevent0: New version of libevent is available from upstream > (75 days old) > > > And I think my debs don't have epoll support. As I recall, you had to > do some header trickery? If you'd send me more information I can update > them, or send you the stuff you need to build the debs yourself (it's > trivial). The header trickery information is now in CVS, in the BUILD file of memcached's directory. (if we're talking about the same kind of header trickery, which I'm not 100% sure) -- avva@livejournal.com From brad@danga.com Tue Aug 26 03:58:52 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 25 Aug 2003 19:58:52 -0700 (PDT) Subject: spinning bug In-Reply-To: <20030826025049.GA17719@pobox.com> References: <20030826025049.GA17719@pobox.com> Message-ID: [ moved to mailing list ] But, how do we tell libevent we want edge-triggered? If we do it ourselves, how do we get libevent's current mode? Should we fork libevent and include it in memcached? (perfectly legal, but questionably immoral) - Brad On Tue, 26 Aug 2003, Anatoly Vorobey wrote: > I'm still racking my brains over this bug, and coming up with nothing > all the time. I have one new crazy idea though. > > Look. We suspect that the spinning bug occurs because some > event starts signalling all the time, and our state machine > doesn't know how to satisfy it, so it keeps signalling, stealing > CPU time. Right? > > But what if we go from level-triggered epoll to edge-triggered? IIRC, > the only difference between them is this: > > if you get a read event and don't read all the data, you won't > get the signal ever again after you return. > > But we always *do* read the data all the way... don't we? We don't > give up and return until we get a result from read() saying we have > nothing else to read (or we go into write mode, of course... but when we > eventually come out of that, we go into the read mode and attempt > read()ing right away w/o relying on being signalled first). > > So, IIRC, by just going into edge-triggered mode we can make the > spinning problem disappear. The "runaway" event won't signal more than > once if we don't handle it, which we apparently don't. Does this sound > logical to you? > > (level-triggered epoll can do edge-triggered as well, so we don't have > to change the patches... just add an ioctl call or whatever) > > -- > avva > > From mellon@pobox.com Tue Aug 26 04:14:36 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Tue, 26 Aug 2003 06:14:36 +0300 Subject: spinning bug In-Reply-To: References: <20030826025049.GA17719@pobox.com> Message-ID: <20030826031436.GA17813@pobox.com> On Mon, Aug 25, 2003 at 07:58:52PM -0700, Brad Fitzpatrick wrote: > [ moved to mailing list ] > > But, how do we tell libevent we want edge-triggered? If we do it > ourselves, how do we get libevent's current mode? > > Should we fork libevent and include it in memcached? (perfectly legal, > but questionably immoral) > > - Brad We could talk to libevent's maintainer about extending libevent with the ability to to a) find out at runtime which method is used (this is useful anyway) and b) set method-specific flags, like an edge_triggered flag for the epoll method. The actual coding needed for this is simple, the question is only whether the maintainer'll agree this is a good thing. I could write the patch for libevent if we get the maintainer to go along with it [however, I do think we need to test this idea locally first; it's super-easy to just augment libevent's epoll_ctl() calls for creation/modification of an event and add an EPOLLET flag to them. We could do that, rebuild libevent, rebuild memcached statically and test the hell out of it. We've never tried edge-triggered epoll before - it's just an idea I had, and for all I know there may be gotchas lurking that I'm not seeing right now]. Alternatively, we could abstract our event handling a bit and make it libevent-independent; detect epoll ourselves and use it if it's present, fall back on libevent otherwise. Forking libevent is yucky and probably the worst alternative. -- avva@livejournal.com From brad@danga.com Tue Aug 26 07:28:50 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Mon, 25 Aug 2003 23:28:50 -0700 (PDT) Subject: spinning bug In-Reply-To: <20030826031436.GA17813@pobox.com> References: <20030826025049.GA17719@pobox.com> <20030826031436.GA17813@pobox.com> Message-ID: Well, try it out. I'll give it stress testing if you get something passing your local testing. On Tue, 26 Aug 2003, Anatoly Vorobey wrote: > On Mon, Aug 25, 2003 at 07:58:52PM -0700, Brad Fitzpatrick wrote: > > [ moved to mailing list ] > > > > But, how do we tell libevent we want edge-triggered? If we do it > > ourselves, how do we get libevent's current mode? > > > > Should we fork libevent and include it in memcached? (perfectly legal, > > but questionably immoral) > > > > - Brad > > We could talk to libevent's maintainer about extending libevent > with the ability to to a) find out at runtime which method is used (this > is useful anyway) and b) set method-specific flags, like > an edge_triggered flag for the epoll method. The actual coding needed > for this is simple, the question is only whether the maintainer'll agree > this is a good thing. I could write the patch for libevent if we get the > maintainer to go along with it [however, I do think we need to test this > idea locally first; it's super-easy to just augment libevent's > epoll_ctl() calls for creation/modification of an event and add an > EPOLLET flag to them. We could do that, rebuild libevent, rebuild > memcached statically and test the hell out of it. We've never tried > edge-triggered epoll before - it's just an idea I had, and for all > I know there may be gotchas lurking that I'm not seeing right now]. > > Alternatively, we could abstract our event handling a bit and make it > libevent-independent; detect epoll ourselves and use it if it's > present, fall back on libevent otherwise. > > Forking libevent is yucky and probably the worst alternative. > > -- > avva@livejournal.com > > From lisa@gentoo.org Wed Aug 27 19:24:10 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Wed, 27 Aug 2003 14:24:10 -0400 Subject: Memcached TODO list? In-Reply-To: References: <1061867976.1789.3.camel@lisa.thedoh.com> <1061919003.2780.211.camel@lisa.thedoh.com> Message-ID: <1062008650.19050.34.camel@lisa.thedoh.com> --=-ZMHVwhFL/cXjxe6mmsgA Content-Type: multipart/mixed; boundary="=-CuBexLMenmHG8ggw7C8p" --=-CuBexLMenmHG8ggw7C8p Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Tue, 2003-08-26 at 14:54, Brad Fitzpatrick wrote: > Well, I'm sending this mail from danga.com. Port 2401 looks open and > happy. Thanks, Brad. Seems to be up now (I was able to find the cvs info from Google). Attached is a patch that will drop root privileges on start up. Affected files: Makefile.am memcached.c Added files: errors.h setuid.h setuid.c I'll admit my C isn't that strong, but this seems to be a good start. The user that will switched to is defined in setuid.h. Any error in the setuid process is given an error label that is defined in errors.h (seems like as good a place as any to put error labels and possibly strings). If the daemon is unable to switch users, it will print an error message (possibly two, if the user doesn't exist) and continue running as normal. Switch users by calling `int switch_user()'. --=20 Regards, -Lisa --=-CuBexLMenmHG8ggw7C8p Content-Disposition: attachment; filename=setuid.patch Content-Type: text/x-patch; name=setuid.patch; charset=ISO-8859-1 Content-Transfer-Encoding: base64 Q29tbW9uIHN1YmRpcmVjdG9yaWVzOiBjdnMvLmRlcHMgYW5kIGN2cy1kaWZmLy5kZXBzDQpDb21t b24gc3ViZGlyZWN0b3JpZXM6IGN2cy9DVlMgYW5kIGN2cy1kaWZmL0NWUw0KZGlmZiAtdU4gY3Zz L01ha2VmaWxlLmFtIGN2cy1kaWZmL01ha2VmaWxlLmFtDQotLS0gY3ZzL01ha2VmaWxlLmFtCTIw MDMtMDgtMjcgMTQ6MDE6NDguMDAwMDAwMDAwIC0wNDAwDQorKysgY3ZzLWRpZmYvTWFrZWZpbGUu YW0JMjAwMy0wOC0yNyAxMzo0ODozMS4wMDAwMDAwMDAgLTA0MDANCkBAIC0xLDYgKzEsNiBAQA0K IGJpbl9QUk9HUkFNUyA9IG1lbWNhY2hlZA0KIA0KLW1lbWNhY2hlZF9TT1VSQ0VTID0gbWVtY2Fj aGVkLmMgc2xhYnMuYyBpdGVtcy5jIG1lbWNhY2hlZC5oIGFzc29jLmMNCittZW1jYWNoZWRfU09V UkNFUyA9IG1lbWNhY2hlZC5jIHNsYWJzLmMgaXRlbXMuYyBtZW1jYWNoZWQuaCBhc3NvYy5jIHNl dHVpZC5jIHNldHVpZC5oIGVycm9ycy5oDQogDQogRElTVF9TVUJESVJTID0gZG9jDQogRVhUUkFf RElTVCA9IGRvYyBUT0RPDQpDb21tb24gc3ViZGlyZWN0b3JpZXM6IGN2cy9hcGkgYW5kIGN2cy1k aWZmL2FwaQ0KQ29tbW9uIHN1YmRpcmVjdG9yaWVzOiBjdnMvZG9jIGFuZCBjdnMtZGlmZi9kb2MN CmRpZmYgLXVOIGN2cy9lcnJvcnMuaCBjdnMtZGlmZi9lcnJvcnMuaA0KLS0tIGN2cy9lcnJvcnMu aAkxOTY5LTEyLTMxIDE5OjAwOjAwLjAwMDAwMDAwMCAtMDUwMA0KKysrIGN2cy1kaWZmL2Vycm9y cy5oCTIwMDMtMDgtMjcgMTQ6MDI6MDkuMDAwMDAwMDAwIC0wNDAwDQpAQCAtMCwwICsxLDggQEAN CisvKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqDQorICogICAgICAgICAgICBlcnJvcnMuaA0KKyAqDQorICog IFdlZCBBdWcgMjcgMTQ6MDE6MTYgMjAwMw0KKyAqICBMaXNhIFNlZWx5ZSA8bGlzYUBnZW50b28u b3JnPiwgZm9yIG1lbWNhY2hlZA0KKyAqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqLw0KKw0KKyNkZWZpbmUg TUNEX0VSUk9SX1NFVFVJRCAxMDANCmRpZmYgLXVOIGN2cy9tZW1jYWNoZWQuYyBjdnMtZGlmZi9t ZW1jYWNoZWQuYw0KLS0tIGN2cy9tZW1jYWNoZWQuYwkyMDAzLTA4LTI3IDE0OjAxOjQ4LjAwMDAw MDAwMCAtMDQwMA0KKysrIGN2cy1kaWZmL21lbWNhY2hlZC5jCTIwMDMtMDgtMjcgMTM6NDg6MzEu MDAwMDAwMDAwIC0wNDAwDQpAQCAtNDcsNiArNDcsNyBAQA0KICNlbmRpZg0KIA0KICNpbmNsdWRl ICJtZW1jYWNoZWQuaCINCisjaW5jbHVkZSAic2V0dWlkLmgiDQogDQogc3RydWN0IHN0YXRzIHN0 YXRzOw0KIHN0cnVjdCBzZXR0aW5ncyBzZXR0aW5nczsNCkBAIC0xMjQ5LDkgKzEyNTAsMTYgQEAN CiAgICAgc3RydWN0IGluX2FkZHIgYWRkcjsNCiAgICAgaW50IGxvY2tfbWVtb3J5ID0gMDsNCiAg ICAgaW50IGRhZW1vbml6ZSA9IDA7DQorCWludCBzZXR1aWRfZXJyb3IgPSAwOw0KIA0KKyAgICAv KiBEcm9wIHJvb3QgcHJpdmVsZWRnZXMgaWYgd2UncmUgcnVubmluZyBhcyByb290ICovDQorCWlm ICggKHNldHVpZF9lcnJvciA9IHN3aXRjaF91c2VyKCkpICE9IDAgKSB7DQorCQkvKiBXZSBoYWQg YW4gZXJyb3IgaW4gZHJvcHBpbmcgdG8gYSB1c2VyKi8NCisJCWZwcmludGYoc3RkZXJyLCJFcnJv ciAoJWQpOiBGYWlsZWQgdG8gZHJvcCByb290IHByaXZpbGVkZ2VzXG4iLHNldHVpZF9lcnJvcik7 DQorCX0NCisJCQ0KICAgICAvKiBpbml0IHNldHRpbmdzICovDQotICAgIHNldHRpbmdzX2luaXQo KTsNCisJc2V0dGluZ3NfaW5pdCgpOw0KIA0KICAgICAvKiBwcm9jZXNzIGFyZ3VtZW50cyAqLw0K ICAgICB3aGlsZSAoKGMgPSBnZXRvcHQoYXJnYywgYXJndiwgInA6bTpjOmtoaXZkbDoiKSkgIT0g LTEpIHsNCkBAIC0xMzQzLDQgKzEzNTEsMyBAQA0KIA0KICAgICByZXR1cm4gMDsNCiB9DQotDQpk aWZmIC11TiBjdnMvc2V0dWlkLmMgY3ZzLWRpZmYvc2V0dWlkLmMNCi0tLSBjdnMvc2V0dWlkLmMJ MTk2OS0xMi0zMSAxOTowMDowMC4wMDAwMDAwMDAgLTA1MDANCisrKyBjdnMtZGlmZi9zZXR1aWQu YwkyMDAzLTA4LTI3IDE0OjEwOjQ1LjAwMDAwMDAwMCAtMDQwMA0KQEAgLTAsMCArMSw3NiBAQA0K Ky8qKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioNCisgKiAgICAgICAgICAgIHNldHVpZC5jDQorICoNCisgKiAg V2VkIEF1ZyAyNyAxMzo1OTozNiAyMDAzDQorICogIExpc2EgU2VlbHllIDxsaXNhQGdlbnRvby5v cmc+LCBmb3IgbWVtY2FjaGVkDQorICoqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKiovDQorIA0KKyNpbmNsdWRl IDxzdGRpby5oPg0KKyNpbmNsdWRlIDxzeXMvdHlwZXMuaD4NCisjaW5jbHVkZSA8dW5pc3RkLmg+ DQorI2luY2x1ZGUgPHN0ZGxpYi5oPg0KKyNpbmNsdWRlIDxwd2QuaD4NCisjaW5jbHVkZSA8ZXJy bm8uaD4NCisNCisjaW5jbHVkZSAic2V0dWlkLmgiDQorI2luY2x1ZGUgImVycm9ycy5oIg0KK2lu dCBzd2l0Y2hfdXNlcigpOw0KK3N0YXRpYyB2b2lkIGdldF9wcmVmZXJyZWRfaWQodWlkX3QgKnVp ZCwgZ2lkX3QgKmdpZCk7DQorDQorDQorDQorc3RhdGljIHZvaWQgZ2V0X3ByZWZlcnJlZF9pZCh1 aWRfdCAqdWlkLCBnaWRfdCAqZ2lkKSB7DQorCS8qIFNldCB1aWQgYW5kIGdpZCB0byB0aGUgcHJl ZmVycmVkIHVzZXIgKGZvdW5kIGluIHNldHVpZC5oKS4gQ2FuIGVpdGhlciBiZQ0KKwkgKiBudW1l cmljIG9yIGEgc3RyaW5nLCBmb3VuZCBpbiAvZXRjL3Bhc3N3ZA0KKwkqLw0KKw0KKwlzdHJ1Y3Qg cGFzc3dkICpwdzsNCisJaWYgKChwdyA9IGdldHB3bmFtKFBVU0VSSUQpKSkgew0KKwkJLy8gdGhp cyBpcyBwcmVmZXJyZWQgdXNlcg0KKwkJKnVpZCA9IHB3LT5wd191aWQ7DQorCQkqZ2lkID0gcHct PnB3X2dpZDsNCisJCXJldHVybjsNCisJDQorCX0NCisJZnByaW50ZihzdGRlcnIsIkVycm9yOiBO byBzdWNoIHVzZXIgJXNcbiIsUFVTRVJJRCk7DQorCWlmICgocHcgPSBnZXRwd25hbSgibm9ib2R5 IikpKSB7DQorCQkvLyB0aGlzIGlzIHRoZSBmYWxsYmFjayB1c2VyDQorCQkqdWlkID0gcHctPnB3 X3VpZDsNCisJCSpnaWQgPSBwdy0+cHdfZ2lkOw0KKwkJcmV0dXJuOw0KKwl9DQorCS8vIHRoaXMg aXMgdGhlIGZhbGwtZmFsbGJhY2sgdXNlciAodGhlcmUgaXMgbm8gJ25vYm9keScgb24gdGhpcyBz eXN0ZW0/PykNCisJKnVpZCA9IDY1NTM0Ow0KKwkqZ2lkID0gNjU1MzQ7DQorCXJldHVybjsNCit9 DQorDQorDQoraW50IHN3aXRjaF91c2VyKCkgew0KKwkvKiBJZiB3ZSBhcmUgcm9vdCwgd2Ugc2V0 dWlkOnNldGdpZCB0byBhIHByZWZlcnJlZCB1c2VyIChmb3VuZCBpbiB0aGUgLmgpDQorCSAqIElm IHdlIGZhaWwgdG8gc2V0dWlkOnNldGdpZCwgd2UnbGwgdHJ5IG5vYm9keTpub2JvZHksIGlmIHRo YXQgZmFpbHMgd2UnbGwNCisJICogdHJ5IHNvbWV0aGluZyBlbHNlICg2NTUzNDo2NTUzNCkNCisJ ICogSWYgd2UncmUgKk5PVCogcm9vdCwgd2UnbGwgbm90IGRvIGFueXRoaW5nLg0KKwkqLw0KKwl1 aWRfdCB1aWQ7IGdpZF90IGdpZDsNCisNCisJZ2V0X3ByZWZlcnJlZF9pZCgmdWlkLCZnaWQpOw0K KwlpZiAoIGdldHVpZCgpICE9IDAgJiYgZ2V0Z2lkKCkgIT0gMCkgew0KKwkJLyogV2UncmUgbm90 IHJvb3QhIERvbid0IGJvdGhlciBzd2l0Y2hpbmchICovDQorCQlyZXR1cm4gMDsNCisJfQ0KKwlp ZiAoc2V0Z2lkKGdpZCkpIA0KKwkJcmV0dXJuIE1DRF9FUlJPUl9TRVRVSUQ7DQorCWlmIChzZXR1 aWQodWlkKSkNCisJCXJldHVybiBNQ0RfRVJST1JfU0VUVUlEOw0KKwkvKiBXZSBzaG91bGQgaGF2 ZSBkcm9wcGVkIGFsbCByb290IHBlcm1pc3Npb25zIGJ5IG5vdyAqLw0KKwkNCisJaWYgKCBnZXR1 aWQoKSA9PSAwIHx8IGdldGdpZCgpID09IDApIHsNCisJCS8qIFdlIEZBSUxFRCB0byBkcm9wIHBl cm1pc3Npb25zIChzZXR1aWQoKSBvciBzZXRnaWQoKSkgb24gdGhpcyBzeXN0ZW0NCisJCSAqIGFy ZSBOT1Qgd29ya2luZyBwcm9wZXJseSENCisJCSovDQorCQlyZXR1cm4gTUNEX0VSUk9SX1NFVFVJ RDsNCisJfQ0KKwkvKiBBdCB0aGlzIHBvaW50LCB3ZSdyZSBzdXJlIHdlIGFyZSBub3Qgcm9vdDpy b290LiAqLw0KKwlyZXR1cm4gMDsNCit9DQpkaWZmIC11TiBjdnMvc2V0dWlkLmggY3ZzLWRpZmYv c2V0dWlkLmgNCi0tLSBjdnMvc2V0dWlkLmgJMTk2OS0xMi0zMSAxOTowMDowMC4wMDAwMDAwMDAg LTA1MDANCisrKyBjdnMtZGlmZi9zZXR1aWQuaAkyMDAzLTA4LTI3IDE0OjExOjAxLjAwMDAwMDAw MCAtMDQwMA0KQEAgLTAsMCArMSw5IEBADQorLyoqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKg0KKyAqICAgICAg ICAgICAgc2V0dWlkLmgNCisgKg0KKyAqICBXZWQgQXVnIDI3IDE0OjAwOjEwIDIwMDMNCisgKiAg TGlzYSBTZWVseWUgPGxpc2FAZ2VudG9vLm9yZz4sIGZvciBtZW1jYWNoZWQNCisgKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKi8NCisNCisNCisjZGVmaW5lIFBVU0VSSUQgIm1lbWNhY2hlZCINCkNvbW1vbiBz dWJkaXJlY3RvcmllczogY3ZzL3dlYnNpdGUgYW5kIGN2cy1kaWZmL3dlYnNpdGUNCm== --=-CuBexLMenmHG8ggw7C8p-- --=-ZMHVwhFL/cXjxe6mmsgA Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.3 (GNU/Linux) iD8DBQA/TPdKy0a1Vh5Jb8URAsAlAKCL1z/sBCktUzgUGBKoRwLjHQSIzQCbBX1+ 6L3r5S7vbilwL28jeMQ8Gt4= =D36B -----END PGP SIGNATURE----- --=-ZMHVwhFL/cXjxe6mmsgA-- From lisa@gentoo.org Wed Aug 27 19:55:47 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Wed, 27 Aug 2003 14:55:47 -0400 Subject: Memcached TODO list? In-Reply-To: <1062008650.19050.34.camel@lisa.thedoh.com> References: <1061867976.1789.3.camel@lisa.thedoh.com> <1061919003.2780.211.camel@lisa.thedoh.com> <1062008650.19050.34.camel@lisa.thedoh.com> Message-ID: <1062010546.19050.41.camel@lisa.thedoh.com> --=-SdEeIDAlz8JwaeQqXxjO Content-Type: multipart/mixed; boundary="=-0BibJWWxZ15pbsJJr1T/" --=-0BibJWWxZ15pbsJJr1T/ Content-Type: text/plain Content-Transfer-Encoding: quoted-printable Ack. I forgot I took out the need for errno.h, here's a better patch that removes it. --=20 Regards, -Lisa --=-0BibJWWxZ15pbsJJr1T/ Content-Disposition: attachment; filename=setuid.patch Content-Transfer-Encoding: base64 Content-Type: text/x-patch; name=setuid.patch; charset=ISO-8859-1 Q29tbW9uIHN1YmRpcmVjdG9yaWVzOiBjdnMvLmRlcHMgYW5kIGN2cy1kaWZmLy5kZXBzDQpDb21t b24gc3ViZGlyZWN0b3JpZXM6IGN2cy9DVlMgYW5kIGN2cy1kaWZmL0NWUw0KZGlmZiAtdU4gY3Zz L01ha2VmaWxlLmFtIGN2cy1kaWZmL01ha2VmaWxlLmFtDQotLS0gY3ZzL01ha2VmaWxlLmFtCTIw MDMtMDgtMjcgMTQ6NTI6MjkuMDAwMDAwMDAwIC0wNDAwDQorKysgY3ZzLWRpZmYvTWFrZWZpbGUu YW0JMjAwMy0wOC0yNyAxMzo0ODozMS4wMDAwMDAwMDAgLTA0MDANCkBAIC0xLDYgKzEsNiBAQA0K IGJpbl9QUk9HUkFNUyA9IG1lbWNhY2hlZA0KIA0KLW1lbWNhY2hlZF9TT1VSQ0VTID0gbWVtY2Fj aGVkLmMgc2xhYnMuYyBpdGVtcy5jIG1lbWNhY2hlZC5oIGFzc29jLmMNCittZW1jYWNoZWRfU09V UkNFUyA9IG1lbWNhY2hlZC5jIHNsYWJzLmMgaXRlbXMuYyBtZW1jYWNoZWQuaCBhc3NvYy5jIHNl dHVpZC5jIHNldHVpZC5oIGVycm9ycy5oDQogDQogRElTVF9TVUJESVJTID0gZG9jDQogRVhUUkFf RElTVCA9IGRvYyBUT0RPDQpDb21tb24gc3ViZGlyZWN0b3JpZXM6IGN2cy9hcGkgYW5kIGN2cy1k aWZmL2FwaQ0KQ29tbW9uIHN1YmRpcmVjdG9yaWVzOiBjdnMvZG9jIGFuZCBjdnMtZGlmZi9kb2MN CmRpZmYgLXVOIGN2cy9lcnJvcnMuaCBjdnMtZGlmZi9lcnJvcnMuaA0KLS0tIGN2cy9lcnJvcnMu aAkxOTY5LTEyLTMxIDE5OjAwOjAwLjAwMDAwMDAwMCAtMDUwMA0KKysrIGN2cy1kaWZmL2Vycm9y cy5oCTIwMDMtMDgtMjcgMTQ6MDI6MDkuMDAwMDAwMDAwIC0wNDAwDQpAQCAtMCwwICsxLDggQEAN CisvKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqDQorICogICAgICAgICAgICBlcnJvcnMuaA0KKyAqDQorICog IFdlZCBBdWcgMjcgMTQ6MDE6MTYgMjAwMw0KKyAqICBMaXNhIFNlZWx5ZSA8bGlzYUBnZW50b28u b3JnPiwgZm9yIG1lbWNhY2hlZA0KKyAqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqLw0KKw0KKyNkZWZpbmUg TUNEX0VSUk9SX1NFVFVJRCAxMDANCmRpZmYgLXVOIGN2cy9tZW1jYWNoZWQuYyBjdnMtZGlmZi9t ZW1jYWNoZWQuYw0KLS0tIGN2cy9tZW1jYWNoZWQuYwkyMDAzLTA4LTI3IDE0OjUyOjMwLjAwMDAw MDAwMCAtMDQwMA0KKysrIGN2cy1kaWZmL21lbWNhY2hlZC5jCTIwMDMtMDgtMjcgMTM6NDg6MzEu MDAwMDAwMDAwIC0wNDAwDQpAQCAtNDcsNiArNDcsNyBAQA0KICNlbmRpZg0KIA0KICNpbmNsdWRl ICJtZW1jYWNoZWQuaCINCisjaW5jbHVkZSAic2V0dWlkLmgiDQogDQogc3RydWN0IHN0YXRzIHN0 YXRzOw0KIHN0cnVjdCBzZXR0aW5ncyBzZXR0aW5nczsNCkBAIC0xMjQ5LDkgKzEyNTAsMTYgQEAN CiAgICAgc3RydWN0IGluX2FkZHIgYWRkcjsNCiAgICAgaW50IGxvY2tfbWVtb3J5ID0gMDsNCiAg ICAgaW50IGRhZW1vbml6ZSA9IDA7DQorCWludCBzZXR1aWRfZXJyb3IgPSAwOw0KIA0KKyAgICAv KiBEcm9wIHJvb3QgcHJpdmVsZWRnZXMgaWYgd2UncmUgcnVubmluZyBhcyByb290ICovDQorCWlm ICggKHNldHVpZF9lcnJvciA9IHN3aXRjaF91c2VyKCkpICE9IDAgKSB7DQorCQkvKiBXZSBoYWQg YW4gZXJyb3IgaW4gZHJvcHBpbmcgdG8gYSB1c2VyKi8NCisJCWZwcmludGYoc3RkZXJyLCJFcnJv ciAoJWQpOiBGYWlsZWQgdG8gZHJvcCByb290IHByaXZpbGVkZ2VzXG4iLHNldHVpZF9lcnJvcik7 DQorCX0NCisJCQ0KICAgICAvKiBpbml0IHNldHRpbmdzICovDQotICAgIHNldHRpbmdzX2luaXQo KTsNCisJc2V0dGluZ3NfaW5pdCgpOw0KIA0KICAgICAvKiBwcm9jZXNzIGFyZ3VtZW50cyAqLw0K ICAgICB3aGlsZSAoKGMgPSBnZXRvcHQoYXJnYywgYXJndiwgInA6bTpjOmtoaXZkbDoiKSkgIT0g LTEpIHsNCkBAIC0xMzQzLDQgKzEzNTEsMyBAQA0KIA0KICAgICByZXR1cm4gMDsNCiB9DQotDQpk aWZmIC11TiBjdnMvc2V0dWlkLmMgY3ZzLWRpZmYvc2V0dWlkLmMNCi0tLSBjdnMvc2V0dWlkLmMJ MTk2OS0xMi0zMSAxOTowMDowMC4wMDAwMDAwMDAgLTA1MDANCisrKyBjdnMtZGlmZi9zZXR1aWQu YwkyMDAzLTA4LTI3IDE0OjUzOjM2LjAwMDAwMDAwMCAtMDQwMA0KQEAgLTAsMCArMSw3NSBAQA0K Ky8qKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioNCisgKiAgICAgICAgICAgIHNldHVpZC5jDQorICoNCisgKiAg V2VkIEF1ZyAyNyAxMzo1OTozNiAyMDAzDQorICogIExpc2EgU2VlbHllIDxsaXNhQGdlbnRvby5v cmc+LCBmb3IgbWVtY2FjaGVkDQorICoqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKiovDQorDQorI2luY2x1ZGUg PHN0ZGlvLmg+IA0KKyNpbmNsdWRlIDxzeXMvdHlwZXMuaD4NCisjaW5jbHVkZSA8dW5pc3RkLmg+ DQorI2luY2x1ZGUgPHN0ZGxpYi5oPg0KKyNpbmNsdWRlIDxwd2QuaD4NCisNCisjaW5jbHVkZSAi c2V0dWlkLmgiDQorI2luY2x1ZGUgImVycm9ycy5oIg0KK2ludCBzd2l0Y2hfdXNlcigpOw0KK3N0 YXRpYyB2b2lkIGdldF9wcmVmZXJyZWRfaWQodWlkX3QgKnVpZCwgZ2lkX3QgKmdpZCk7DQorDQor DQorDQorc3RhdGljIHZvaWQgZ2V0X3ByZWZlcnJlZF9pZCh1aWRfdCAqdWlkLCBnaWRfdCAqZ2lk KSB7DQorCS8qIFNldCB1aWQgYW5kIGdpZCB0byB0aGUgcHJlZmVycmVkIHVzZXIgKGZvdW5kIGlu IHNldHVpZC5oKS4gQ2FuIGVpdGhlciBiZQ0KKwkgKiBudW1lcmljIG9yIGEgc3RyaW5nLCBmb3Vu ZCBpbiAvZXRjL3Bhc3N3ZA0KKwkqLw0KKw0KKwlzdHJ1Y3QgcGFzc3dkICpwdzsNCisJaWYgKChw dyA9IGdldHB3bmFtKFBVU0VSSUQpKSkgew0KKwkJLy8gdGhpcyBpcyBwcmVmZXJyZWQgdXNlcg0K KwkJKnVpZCA9IHB3LT5wd191aWQ7DQorCQkqZ2lkID0gcHctPnB3X2dpZDsNCisJCXJldHVybjsN CisJDQorCX0NCisJZnByaW50ZihzdGRlcnIsIkVycm9yOiBObyBzdWNoIHVzZXIgJXNcbiIsUFVT RVJJRCk7DQorCWlmICgocHcgPSBnZXRwd25hbSgibm9ib2R5IikpKSB7DQorCQkvLyB0aGlzIGlz IHRoZSBmYWxsYmFjayB1c2VyDQorCQkqdWlkID0gcHctPnB3X3VpZDsNCisJCSpnaWQgPSBwdy0+ cHdfZ2lkOw0KKwkJcmV0dXJuOw0KKwl9DQorCS8vIHRoaXMgaXMgdGhlIGZhbGwtZmFsbGJhY2sg dXNlciAodGhlcmUgaXMgbm8gJ25vYm9keScgb24gdGhpcyBzeXN0ZW0/PykNCisJKnVpZCA9IDY1 NTM0Ow0KKwkqZ2lkID0gNjU1MzQ7DQorCXJldHVybjsNCit9DQorDQorDQoraW50IHN3aXRjaF91 c2VyKCkgew0KKwkvKiBJZiB3ZSBhcmUgcm9vdCwgd2Ugc2V0dWlkOnNldGdpZCB0byBhIHByZWZl cnJlZCB1c2VyIChmb3VuZCBpbiB0aGUgLmgpDQorCSAqIElmIHdlIGZhaWwgdG8gc2V0dWlkOnNl dGdpZCwgd2UnbGwgdHJ5IG5vYm9keTpub2JvZHksIGlmIHRoYXQgZmFpbHMgd2UnbGwNCisJICog dHJ5IHNvbWV0aGluZyBlbHNlICg2NTUzNDo2NTUzNCkNCisJICogSWYgd2UncmUgKk5PVCogcm9v dCwgd2UnbGwgbm90IGRvIGFueXRoaW5nLg0KKwkqLw0KKwl1aWRfdCB1aWQ7IGdpZF90IGdpZDsN CisNCisJZ2V0X3ByZWZlcnJlZF9pZCgmdWlkLCZnaWQpOw0KKwlpZiAoIGdldHVpZCgpICE9IDAg JiYgZ2V0Z2lkKCkgIT0gMCkgew0KKwkJLyogV2UncmUgbm90IHJvb3QhIERvbid0IGJvdGhlciBz d2l0Y2hpbmchICovDQorCQlyZXR1cm4gMDsNCisJfQ0KKwlpZiAoc2V0Z2lkKGdpZCkpIA0KKwkJ cmV0dXJuIE1DRF9FUlJPUl9TRVRVSUQ7DQorCWlmIChzZXR1aWQodWlkKSkNCisJCXJldHVybiBN Q0RfRVJST1JfU0VUVUlEOw0KKwkvKiBXZSBzaG91bGQgaGF2ZSBkcm9wcGVkIGFsbCByb290IHBl cm1pc3Npb25zIGJ5IG5vdyAqLw0KKwkNCisJaWYgKCBnZXR1aWQoKSA9PSAwIHx8IGdldGdpZCgp ID09IDApIHsNCisJCS8qIFdlIEZBSUxFRCB0byBkcm9wIHBlcm1pc3Npb25zIChzZXR1aWQoKSBv ciBzZXRnaWQoKSkgb24gdGhpcyBzeXN0ZW0NCisJCSAqIGFyZSBOT1Qgd29ya2luZyBwcm9wZXJs eSENCisJCSovDQorCQlyZXR1cm4gTUNEX0VSUk9SX1NFVFVJRDsNCisJfQ0KKwkvKiBBdCB0aGlz IHBvaW50LCB3ZSdyZSBzdXJlIHdlIGFyZSBub3Qgcm9vdDpyb290LiAqLw0KKwlyZXR1cm4gMDsN Cit9DQpkaWZmIC11TiBjdnMvc2V0dWlkLmggY3ZzLWRpZmYvc2V0dWlkLmgNCi0tLSBjdnMvc2V0 dWlkLmgJMTk2OS0xMi0zMSAxOTowMDowMC4wMDAwMDAwMDAgLTA1MDANCisrKyBjdnMtZGlmZi9z ZXR1aWQuaAkyMDAzLTA4LTI3IDE0OjExOjAxLjAwMDAwMDAwMCAtMDQwMA0KQEAgLTAsMCArMSw5 IEBADQorLyoqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKg0KKyAqICAgICAgICAgICAgc2V0dWlkLmgNCisgKg0K KyAqICBXZWQgQXVnIDI3IDE0OjAwOjEwIDIwMDMNCisgKiAgTGlzYSBTZWVseWUgPGxpc2FAZ2Vu dG9vLm9yZz4sIGZvciBtZW1jYWNoZWQNCisgKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKi8NCisNCisNCisj ZGVmaW5lIFBVU0VSSUQgIm1lbWNhY2hlZCINCkNvbW1vbiBzdWJkaXJlY3RvcmllczogY3ZzL3dl YnNpdGUgYW5kIGN2cy1kaWZmL3dlYnNpdGUNCg== --=-0BibJWWxZ15pbsJJr1T/-- --=-SdEeIDAlz8JwaeQqXxjO Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.3 (GNU/Linux) iD8DBQA/TP6yy0a1Vh5Jb8URAuzLAJ4nztlFhpqB6AwO1ljWabmsq5UrnQCg47kF YFmf3w+iCIuHdNbRqI+ItRw= =rUQ1 -----END PGP SIGNATURE----- --=-SdEeIDAlz8JwaeQqXxjO-- From rg@tcslon.com Wed Aug 27 23:04:27 2003 From: rg@tcslon.com (Russell Garrett) Date: Wed, 27 Aug 2003 23:04:27 +0100 Subject: Satisfied user :) Message-ID: Just thought I'd drop a note - we're now using memcached and the PHP client very successfully on www.last.fm and www.audioscrobbler.com (which are far from finished, I might add). It makes it so much easier when you can code PHP without worrying about the overheads of using objects all the time. And it's shockingly easy :). I've got a few tweaks and patches to the PHP client to submit when I've got a free second, which may not be for a while. Cheers, ---------------------------------------------------------------------------- Russ Garrett Developer Last.fm Limited From brad@danga.com Wed Aug 27 23:08:10 2003 From: brad@danga.com (Brad Fitzpatrick) Date: Wed, 27 Aug 2003 15:08:10 -0700 (PDT) Subject: Satisfied user :) In-Reply-To: References: Message-ID: Wonderful! On Wed, 27 Aug 2003, Russell Garrett wrote: > Just thought I'd drop a note - we're now using memcached and the PHP client > very successfully on www.last.fm and www.audioscrobbler.com (which are far > from finished, I might add). It makes it so much easier when you can code > PHP without worrying about the overheads of using objects all the time. And > it's shockingly easy :). > > I've got a few tweaks and patches to the PHP client to submit when I've got > a free second, which may not be for a while. > > Cheers, > > ---------------------------------------------------------------------------- > Russ Garrett > Developer > Last.fm Limited > > From lisa@gentoo.org Thu Aug 28 03:29:28 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Wed, 27 Aug 2003 22:29:28 -0400 Subject: Memcached TODO list? In-Reply-To: <1062010546.19050.41.camel@lisa.thedoh.com> References: <1061867976.1789.3.camel@lisa.thedoh.com> <1061919003.2780.211.camel@lisa.thedoh.com> <1062008650.19050.34.camel@lisa.thedoh.com> <1062010546.19050.41.camel@lisa.thedoh.com> Message-ID: <1062037768.11757.26.camel@lisa.thedoh.com> On Wed, 2003-08-27 at 14:55, Lisa Marie Seelye wrote: > Ack. I forgot I took out the need for errno.h, here's a better patch > that removes it. *sigh* if this isn't text/plain, the patch can be found at http://dev.gentoo.org/~lisa/memcached/setuid.patch Sorry for the binary crap, turns out you can't digitally sign a message and attach a _plain text_ file. -- Regards, -Lisa From jmat@shutdown.net Thu Aug 28 03:54:31 2003 From: jmat@shutdown.net (Justin Matlock) Date: Wed, 27 Aug 2003 22:54:31 -0400 Subject: Satisfied user :) References: Message-ID: <001001c36d0f$b55e77b0$f801010a@nomium.net> Now that I'm back online (yay!), I'm going to rewrite the entire _get routines for the PHP client. streams support was fixed in 4.3.3, and it finally works like it's supposed to. I've been seeing some speed increases using streams vs. individual socket calls. Nothing major, but it's much cleaner... ConEd managed to send a huge power spike into my apartment when they turned the lights back on two weeks back, and blew out everything and anything that had an electrical circuit in it. It fried my UPS's and the three computers that were plugged into them (I think the spike went through the ethernet switch).... My TV literally exploded... It even got the brand new Dell PowerEdge I had received the day before for a new database server... :( .. I made the mistake of not unplugging everything before I left the city on Saturday, and when I got back... well, it wasn't pretty. my insurance is covering everything, they're just incredibly slow because of all of the other claims in the city from that weekend. Anyway, I have a new notebook, and got Gentoo up and running on it this afternoon (if you're using RedHat or Debian, you don't know what you're missing). I should have a test release of the streams version ready by the end of the weekend. I've also written a few example scripts to show people how to better integrate it into their sites using the class (and not be exposed to the class at all, since a lot of people hate objects) :) J ----- Original Message ----- From: "Brad Fitzpatrick" To: Sent: Wednesday, August 27, 2003 6:08 PM Subject: Re: Satisfied user :) > Wonderful! > > > On Wed, 27 Aug 2003, Russell Garrett wrote: > > > Just thought I'd drop a note - we're now using memcached and the PHP client > > very successfully on www.last.fm and www.audioscrobbler.com (which are far > > from finished, I might add). It makes it so much easier when you can code > > PHP without worrying about the overheads of using objects all the time. And > > it's shockingly easy :). > > > > I've got a few tweaks and patches to the PHP client to submit when I've got > > a free second, which may not be for a while. > > > > Cheers, > > > > -------------------------------------------------------------------------- -- > > Russ Garrett > > Developer > > Last.fm Limited > > > > > From mellon@pobox.com Thu Aug 28 10:12:20 2003 From: mellon@pobox.com (Anatoly Vorobey) Date: Thu, 28 Aug 2003 12:12:20 +0300 Subject: Memcached TODO list? In-Reply-To: <1062037768.11757.26.camel@lisa.thedoh.com> References: <1061867976.1789.3.camel@lisa.thedoh.com> <1061919003.2780.211.camel@lisa.thedoh.com> <1062008650.19050.34.camel@lisa.thedoh.com> <1062010546.19050.41.camel@lisa.thedoh.com> <1062037768.11757.26.camel@lisa.thedoh.com> Message-ID: <20030828091220.GA17343@pobox.com> Lisa, You wrote on Wed, Aug 27, 2003 at 10:29:28PM -0400: > On Wed, 2003-08-27 at 14:55, Lisa Marie Seelye wrote: > > Ack. I forgot I took out the need for errno.h, here's a better patch > > that removes it. > > *sigh* if this isn't text/plain, the patch can be found at > http://dev.gentoo.org/~lisa/memcached/setuid.patch > > Sorry for the binary crap, turns out you can't digitally sign a message > and attach a _plain text_ file. It's a nice patch. However, I can't help thinking it might be better to keep it simple... or simpl-er, anyway. Namely, - have a command-line switch "-u user" meaning "assume identity of user when run as root" - refuse to run as root unless the switch is given. If the user running memcached is careless enough to want to run it as root w/o providing the switch, w/o reading the README file (where it should be mentioned), etc., we might as well fail and draw the user's attention to that. This will have the benefits of - avoiding hardcoded stuff in the code (in your patch, three pieces of hardcoded information: one username and two fallbacks) - reduce the code necessary to 6-7 lines or so in the main .c file w/o requiring another source and header file just for this feature. What do you think? -- avva@livejournal.com From lisa@gentoo.org Thu Aug 28 20:04:29 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Thu, 28 Aug 2003 15:04:29 -0400 Subject: Memcached TODO list? In-Reply-To: <20030828091220.GA17343@pobox.com> References: <1061867976.1789.3.camel@lisa.thedoh.com> <1061919003.2780.211.camel@lisa.thedoh.com> <1062008650.19050.34.camel@lisa.thedoh.com> <1062010546.19050.41.camel@lisa.thedoh.com> <1062037768.11757.26.camel@lisa.thedoh.com> <20030828091220.GA17343@pobox.com> Message-ID: <1062097468.11755.33.camel@lisa.thedoh.com> --=-ek/EzJPcidgPNEmJAps0 Content-Type: text/plain Content-Transfer-Encoding: quoted-printable On Thu, 2003-08-28 at 05:12, Anatoly Vorobey wrote: > Lisa, > It's a nice patch. However, I can't help thinking it might be better > to keep it simple... or simpl-er, anyway. Namely, >=20 > - have a command-line switch "-u user" meaning "assume identity of user =20 > when run as root" > - refuse to run as root unless the switch is given. >=20 > If the user running memcached is careless enough to want to run it as > root w/o providing the switch, w/o reading the README file (where it > should be mentioned), etc., we might as well fail and draw the user's > attention to that.=20 >=20 > This will have the benefits of >=20 > - avoiding hardcoded stuff in the code (in your patch, three pieces > of hardcoded information: one username and two fallbacks) > - reduce the code necessary to 6-7 lines or so in the main .c file > w/o requiring another source and header file just for this feature. >=20 > What do you think? That's a better idea ;-). But I like the idea of keeping the setuid code in its own file. Not having to draw in a setuid.h is a good thing, since it keeps things simpler. The user flag will still be in the memcached.c file. I'll see about whipping up a -u|--user flag replacement. --=20 Regards, -Lisa --=-ek/EzJPcidgPNEmJAps0 Content-Type: application/pgp-signature; name=signature.asc Content-Description: This is a digitally signed message part -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.3 (GNU/Linux) iD8DBQA/TlI8y0a1Vh5Jb8URAlbiAKDkqnVpbYdYEqMLWsYkN4U3V3Oe1QCfX5ON wAXvMs1lYANe1apRrIawZ8o= =Kcdy -----END PGP SIGNATURE----- --=-ek/EzJPcidgPNEmJAps0-- From lisa@gentoo.org Fri Aug 29 05:16:18 2003 From: lisa@gentoo.org (Lisa Marie Seelye) Date: Fri, 29 Aug 2003 00:16:18 -0400 Subject: Memcached TODO list? In-Reply-To: <20030828091220.GA17343@pobox.com> References: <1061867976.1789.3.camel@lisa.thedoh.com> <1061919003.2780.211.camel@lisa.thedoh.com> <1062008650.19050.34.camel@lisa.thedoh.com> <1062010546.19050.41.camel@lisa.thedoh.com> <1062037768.11757.26.camel@lisa.thedoh.com> <20030828091220.GA17343@pobox.com> Message-ID: <1062130578.11755.48.camel@lisa.thedoh.com> --=-nCdao2ZCRaW7NKj58fmr Content-Type: text/plain Content-Transfer-Encoding: 7bit On Thu, 2003-08-28 at 05:12, Anatoly Vorobey wrote: > - have a command-line switch "-u user" meaning "assume identity of user > when run as root" > - refuse to run as root unless the switch is given. To this end, here is a revised patch. If it fails to be presented in a text/plain format, it can be found at http://dev.gentoo.org/~lisa/memcached/setuid1.patch (note the '1') How is this? It passes my local tests. Not sure if I got the logic down right... check the comments in the setuid.c file for how I interpreted it. -- Regards, -Lisa --=-nCdao2ZCRaW7NKj58fmr Content-Disposition: attachment; filename=setuid1.patch Content-Type: text/x-patch; name=setuid1.patch; charset=iso-8859-1 Content-Transfer-Encoding: 7bit Common subdirectories: cvs/.deps and cvs-diff/.deps Common subdirectories: cvs/CVS and cvs-diff/CVS diff -uN cvs/Makefile.am cvs-diff/Makefile.am --- cvs/Makefile.am 2003-08-27 14:52:29.000000000 -0400 +++ cvs-diff/Makefile.am 2003-08-27 13:48:31.000000000 -0400 @@ -1,6 +1,6 @@ bin_PROGRAMS = memcached -memcached_SOURCES = memcached.c slabs.c items.c memcached.h assoc.c +memcached_SOURCES = memcached.c slabs.c items.c memcached.h assoc.c setuid.c setuid.h errors.h DIST_SUBDIRS = doc EXTRA_DIST = doc TODO Common subdirectories: cvs/api and cvs-diff/api Common subdirectories: cvs/doc and cvs-diff/doc diff -uN cvs/errors.h cvs-diff/errors.h --- cvs/errors.h 1969-12-31 19:00:00.000000000 -0500 +++ cvs-diff/errors.h 2003-08-28 15:41:23.000000000 -0400 @@ -0,0 +1,10 @@ +/*************************************************************************** + * errors.h + * + * Wed Aug 27 14:01:16 2003 + * Lisa Seelye , for memcached + ****************************************************************************/ + +#define MCD_ERROR_SETUID 100 +#define MCD_RUNNING_AS_ROOT 90 +#define MCD_NO_SPECIFIED_USERNAME 91 diff -uN cvs/memcached.c cvs-diff/memcached.c --- cvs/memcached.c 2003-08-27 14:52:30.000000000 -0400 +++ cvs-diff/memcached.c 2003-08-29 00:10:59.000000000 -0400 @@ -47,6 +47,7 @@ #endif #include "memcached.h" +#include "errors.h" //setuid error codes live within struct stats stats; struct settings settings; @@ -1249,13 +1250,18 @@ struct in_addr addr; int lock_memory = 0; int daemonize = 0; + int setuid_error = 0; + char *username = ""; /* init settings */ - settings_init(); + settings_init(); /* process arguments */ - while ((c = getopt(argc, argv, "p:m:c:khivdl:")) != -1) { + while ((c = getopt(argc, argv, "u:p:m:c:khivdl:")) != -1) { switch (c) { + case 'u': + username = optarg; + break; case 'p': settings.port = atoi(optarg); break; @@ -1338,9 +1344,18 @@ todelete = malloc(sizeof(item *)*deltotal); delete_handler(0,0,0); /* sets up the event */ + /* I think here would be a good place to discard (or attempt to) our + * priveledges. + */ + + setuid_error = switch_user(username); + if (setuid_error) { + fprintf(stderr,"Failed to setuid %s\n",username); + exit(MCD_ERROR_SETUID); + } + /* enter the loop */ event_loop(0); return 0; } - diff -uN cvs/setuid.c cvs-diff/setuid.c --- cvs/setuid.c 1969-12-31 19:00:00.000000000 -0500 +++ cvs-diff/setuid.c 2003-08-29 00:08:33.000000000 -0400 @@ -0,0 +1,105 @@ +/*************************************************************************** + * setuid.c + * + * Wed Aug 27 13:59:36 2003 + * Lisa Seelye , for memcached + ****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +#include "errors.h" //setuid errors live within + +int switch_user(char *username); +static int get_id(char *username, uid_t *uid, gid_t *gid); +int is_superuser(void); + +int is_superuser(void) { + /* quickie to see if we're the superuser or not */ + return ( (getuid() == 0 && getgid() == 0) ); +} + +static int get_id(char *username, uid_t *uid, gid_t *gid) { + /* Set uid and gid to the preferred user (found in setuid.h). Can either be + * numeric or a string, found in /etc/passwd. + */ + struct passwd *pw; + + if ((pw = getpwnam(username))) { + // Name exists + *uid = pw->pw_uid; + *gid = pw->pw_gid; + return 0; + } + /* something Bad happened, so send back an error */ + return MCD_ERROR_SETUID; +} + +int switch_user(char *username) { + /* Set uid and gid to the username specified in the function parameters. + * + * If the user starts memcached as root and fails to specify a username to + * switch to (`-u username'), we will return MCD_NO_SPECIFIED_USERNAME + * + * If memcached is started by a non-root user, we will attempt to switch to + * the username specified (`-u username'), and issue an error if we can't + * MCD_ERROR_SETUID. + * + * If the root user starts memcached and specifies `-u root' we will let + * them run as root. + */ + + uid_t uid; gid_t gid; + + + if (is_superuser()) { + // we are root + if (strcmp(username,"") == 0) { + // username isnt set and we're running as root. let's get outta here + fprintf(stderr,"No username specified with -u option!\n"); + exit(MCD_NO_SPECIFIED_USERNAME); + } + else { + // We're root, but the user gave us a username to switch too. + int retval; + retval = get_id(username,&uid,&gid); + if (retval) { + //fprintf(stderr,"An error occurred while trying to setuid\nExiting...\n"); + return retval; + } // Error in getting uid/gid. + } // we're okay for the setuid/setgid command Down Below + } + else { + //we are not root + if (strcmp(username,"") == 0) { + //no username, so we'll just run as $USER + return 0; + } + else { + /* Here we really have an option to let the user continue running as + * themselves, or to quit outright. I've decided to exit. + */ + int retval; + retval = get_id(username,&uid,&gid); + if (retval) { + //fprintf(stderr,"An error occurred while trying to setuid\nExiting...\n"); + return retval; + } // Error in getting uid/gid. + } + } + if (setgid(gid)) + return MCD_ERROR_SETUID; + if (setuid(uid)) + return MCD_ERROR_SETUID; + + if ((getuid() == 0 || getgid() == 0) && strcmp(username,"root") != 0) { + /* We're still root, and we shouldn't be! */ + return MCD_ERROR_SETUID; + } + return 0; +} Common subdirectories: cvs/website and cvs-diff/website --=-nCdao2ZCRaW7NKj58fmr--