Merge pull request #2462 from BerriAI/litellm_imp_mem_use

[Litellm-Proxy, Router] improve memory usage - don't store messages in memory, previous models in memory

litellm/router.py

@@ -1543,13 +1543,18 @@ class Router:
             ) in (
                 kwargs.items()
             ):  # log everything in kwargs except the old previous_models value - prevent nesting
-                if k != "metadata":
+                if k not in ["metadata", "messages", "original_function"]:
                     previous_model[k] = v
                 elif k == "metadata" and isinstance(v, dict):
                     previous_model["metadata"] = {}  # type: ignore
                     for metadata_k, metadata_v in kwargs["metadata"].items():
                         if metadata_k != "previous_models":
                             previous_model[k][metadata_k] = metadata_v  # type: ignore
+
+            # check current size of self.previous_models, if it's larger than 3, remove the first element
+            if len(self.previous_models) > 3:
+                self.previous_models.pop(0)
+
             self.previous_models.append(previous_model)
             kwargs["metadata"]["previous_models"] = self.previous_models
             return kwargs

litellm/tests/test_mem_usage.py

@@ -101,6 +101,10 @@
 #         print(n, time.time() - start, len(successful_completions))
 #     print()
 #     print(vars(router))
+#     prev_models = router.previous_models
+
+#     print("vars in prev_models")
+#     print(prev_models[0].keys())
 
 
 # if __name__ == "__main__":