Bug 1532955 - Track available memory on linux. r=gsvelto,tkikuchi

This introduces a low memory watcher that dispatches an offthread read of /proc/meminfo every 5000/1000ms depending on memory levels, then determines which information to act on. It works like this:
- Get a percentage of `MemAvailable` versus `MemTotal`.
- If memory drops below 5% availability, we are in a memory pressure scenario
- If `MemAvailable` is not large enough to accommodate a content process, we are in a memory pressure scenario
- If we are in a memory pressure scenario, notify the observers from the main thread.

The value I decided to use to represent a content process was based on observation and should be adjusted if it is not representative of what we consider a "typical" content process.

Differential Revision: https://phabricator.services.mozilla.com/D117972
This commit is contained in:
kriswright 2021-12-02 11:14:30 +00:00
Родитель 56a43f9e9e
Коммит df91a347b4
11 изменённых файлов: 640 добавлений и 2 удалений

Просмотреть файл

@ -46,6 +46,7 @@ MWQThread
MediaCache
MediaTelemetry
MediaTrackGrph
MemoryPoller
mtransport
NamedPipeSrv
Netlink Monitor

Просмотреть файл

@ -1180,7 +1180,7 @@
value: 16777216
mirror: always
#ifdef XP_WIN
#if defined(XP_WIN) || defined(XP_LINUX)
# Notify TabUnloader or send the memory pressure if the memory resource
# notification is signaled AND the available commit space is lower than
# this value.
@ -1190,6 +1190,16 @@
mirror: always
#endif
#ifdef XP_LINUX
# On Linux we also check available memory in comparison to total memory,
# and use this percent value (out of 100) to determine if we are in a
# low memory scenario.
- name: browser.low_commit_space_threshold_percent
type: RelaxedAtomicUint32
value: 5
mirror: always
#endif
# Render animations and videos as a solid color
- name: browser.measurement.render_anims_and_video_solid
type: RelaxedAtomicBool

Просмотреть файл

@ -44,6 +44,8 @@ skip-if = os != 'win'
[TestMacroForEach]
[TestMathAlgorithms]
[TestMaybe]
[TestMemoryPressureWatcherLinux]
skip-if = os != 'linux'
[TestMMPolicy]
skip-if = os != 'win'
[TestNativeNt]

Просмотреть файл

@ -169,7 +169,8 @@ void nsAvailableMemoryWatcherBase::RecordTelemetryEventOnHighMemory() {
// Define the fallback method for a platform for which a platform-specific
// CreateAvailableMemoryWatcher() is not defined.
#if !defined(XP_WIN) && !defined(XP_MACOSX)
#if defined(ANDROID) || \
!defined(XP_WIN) && !defined(XP_MACOSX) && !defined(XP_LINUX)
already_AddRefed<nsAvailableMemoryWatcherBase> CreateAvailableMemoryWatcher() {
RefPtr instance(new nsAvailableMemoryWatcherBase);
return do_AddRef(instance);

Просмотреть файл

@ -0,0 +1,255 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "AvailableMemoryWatcher.h"
#include "AvailableMemoryWatcherUtils.h"
#include "mozilla/Services.h"
#include "mozilla/StaticPrefs_browser.h"
#include "mozilla/Unused.h"
#include "nsAppRunner.h"
#include "nsIObserverService.h"
#include "nsISupports.h"
#include "nsITimer.h"
#include "nsIThread.h"
#include "nsMemoryPressure.h"
namespace mozilla {
// Linux has no native low memory detection. This class creates a timer that
// polls for low memory and sends a low memory notification if it notices a
// memory pressure event.
class nsAvailableMemoryWatcher final : public nsITimerCallback,
public nsINamed,
public nsAvailableMemoryWatcherBase {
public:
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_NSITIMERCALLBACK
NS_DECL_NSIOBSERVER
NS_DECL_NSINAMED
nsresult Init() override;
nsAvailableMemoryWatcher();
void HandleLowMemory();
void MaybeHandleHighMemory();
private:
~nsAvailableMemoryWatcher() = default;
void StartPolling(const MutexAutoLock&);
void StopPolling(const MutexAutoLock&);
void ShutDown(const MutexAutoLock&);
static bool IsMemoryLow();
nsCOMPtr<nsITimer> mTimer;
nsCOMPtr<nsIThread> mThread;
bool mPolling;
bool mUnderMemoryPressure;
// We might tell polling to start/stop from our polling thread
// or from the main thread during ::Observe().
Mutex mMutex;
// Polling interval to check for low memory. In high memory scenarios,
// default to 5000 ms between each check.
static const uint32_t kHighMemoryPollingIntervalMS = 5000;
// Polling interval to check for low memory. Default to 1000 ms between each
// check. Use this interval when memory is low,
static const uint32_t kLowMemoryPollingIntervalMS = 1000;
};
// A modern version of linux should keep memory information in the
// /proc/meminfo path.
static const char* kMeminfoPath = "/proc/meminfo";
nsAvailableMemoryWatcher::nsAvailableMemoryWatcher()
: mPolling(false),
mUnderMemoryPressure(false),
mMutex("Memory Poller mutex") {}
nsresult nsAvailableMemoryWatcher::Init() {
nsresult rv = nsAvailableMemoryWatcherBase::Init();
if (NS_FAILED(rv)) {
return rv;
}
mTimer = NS_NewTimer();
nsCOMPtr<nsIThread> thread;
// We have to make our own thread here instead of using the background pool,
// because some low memory scenarios can cause the background pool to fill.
rv = NS_NewNamedThread("MemoryPoller", getter_AddRefs(thread));
if (NS_FAILED(rv)) {
NS_WARNING("Couldn't make a thread for nsAvailableMemoryWatcher.");
// In this scenario we can't poll for low memory, since we can't dispatch
// to our memory watcher thread.
return rv;
}
mThread = thread;
MutexAutoLock lock(mMutex);
StartPolling(lock);
return NS_OK;
}
already_AddRefed<nsAvailableMemoryWatcherBase> CreateAvailableMemoryWatcher() {
RefPtr watcher(new nsAvailableMemoryWatcher);
if (NS_FAILED(watcher->Init())) {
return do_AddRef(new nsAvailableMemoryWatcherBase);
}
return watcher.forget();
}
NS_IMPL_ISUPPORTS_INHERITED(nsAvailableMemoryWatcher,
nsAvailableMemoryWatcherBase, nsITimerCallback,
nsIObserver);
void nsAvailableMemoryWatcher::StopPolling(const MutexAutoLock&) {
if (mPolling && mTimer) {
// stop dispatching memory checks to the thread.
mTimer->Cancel();
mPolling = false;
}
}
// Check /proc/meminfo for low memory. Largely C method for reading
// /proc/meminfo.
/* static */
bool nsAvailableMemoryWatcher::IsMemoryLow() {
MemoryInfo memInfo{0, 0};
bool aResult = false;
nsresult rv = ReadMemoryFile(kMeminfoPath, memInfo);
if (NS_FAILED(rv) || memInfo.memAvailable == 0) {
// If memAvailable cannot be found, then we are using an older system.
// We can't accurately poll on this.
return aResult;
}
unsigned long memoryAsPercentage =
(memInfo.memAvailable * 100) / memInfo.memTotal;
if (memoryAsPercentage <=
StaticPrefs::browser_low_commit_space_threshold_percent() ||
memInfo.memAvailable <
StaticPrefs::browser_low_commit_space_threshold_mb() * 1024) {
aResult = true;
}
return aResult;
}
void nsAvailableMemoryWatcher::ShutDown(const MutexAutoLock&) {
if (mTimer) {
mTimer->Cancel();
}
if (mThread) {
mThread->Shutdown();
}
}
// We will use this to poll for low memory.
NS_IMETHODIMP
nsAvailableMemoryWatcher::Notify(nsITimer* aTimer) {
MutexAutoLock lock(mMutex);
if (!mThread) {
// If we've made it this far and there's no |mThread|,
// we might have failed to dispatch it for some reason.
MOZ_ASSERT(mThread);
return NS_ERROR_FAILURE;
}
nsresult rv = mThread->Dispatch(
NS_NewRunnableFunction("MemoryPoller", [self = RefPtr{this}]() {
if (self->IsMemoryLow()) {
self->HandleLowMemory();
} else {
self->MaybeHandleHighMemory();
}
}));
if NS_FAILED (rv) {
NS_WARNING("Cannot dispatch memory polling event.");
}
return NS_OK;
}
void nsAvailableMemoryWatcher::HandleLowMemory() {
MutexAutoLock lock(mMutex);
if (!mUnderMemoryPressure) {
mUnderMemoryPressure = true;
// Poll more frequently under memory pressure.
StartPolling(lock);
}
UpdateLowMemoryTimeStamp();
// We handle low memory offthread, but we want to unload
// tabs only from the main thread, so we will dispatch this
// back to the main thread.
NS_DispatchToMainThread(NS_NewRunnableFunction(
"nsAvailableMemoryWatcher::OnLowMemory",
[self = RefPtr{this}]() { self->mTabUnloader->UnloadTabAsync(); }));
}
// If memory is not low, we may need to dispatch an
// event for it if we have been under memory pressure.
// We can also adjust our polling interval.
void nsAvailableMemoryWatcher::MaybeHandleHighMemory() {
MutexAutoLock lock(mMutex);
if (mUnderMemoryPressure) {
RecordTelemetryEventOnHighMemory();
NS_NotifyOfEventualMemoryPressure(MemoryPressureState::NoPressure);
mUnderMemoryPressure = false;
}
StartPolling(lock);
}
// When we change the polling interval, we will need to restart the timer
// on the new interval.
void nsAvailableMemoryWatcher::StartPolling(const MutexAutoLock& aLock) {
uint32_t pollingInterval = mUnderMemoryPressure
? kLowMemoryPollingIntervalMS
: kHighMemoryPollingIntervalMS;
if (!mPolling) {
// Restart the timer with the new interval if it has stopped.
// For testing, use a small polling interval.
if (NS_SUCCEEDED(
mTimer->InitWithCallback(this, gIsGtest ? 10 : pollingInterval,
nsITimer::TYPE_REPEATING_SLACK))) {
mPolling = true;
}
} else {
mTimer->SetDelay(gIsGtest ? 10 : pollingInterval);
}
}
// Observe events for shutting down and starting/stopping the timer.
NS_IMETHODIMP
nsAvailableMemoryWatcher::Observe(nsISupports* aSubject, const char* aTopic,
const char16_t* aData) {
nsresult rv = nsAvailableMemoryWatcherBase::Observe(aSubject, aTopic, aData);
if (NS_FAILED(rv)) {
return rv;
}
MutexAutoLock lock(mMutex);
if (strcmp(aTopic, "xpcom-shutdown") == 0) {
ShutDown(lock);
} else if (strcmp(aTopic, "user-interaction-active") == 0) {
StartPolling(lock);
} else if (strcmp(aTopic, "user-interaction-inactive") == 0) {
StopPolling(lock);
}
return NS_OK;
}
NS_IMETHODIMP nsAvailableMemoryWatcher::GetName(nsACString& aName) {
aName.AssignLiteral("nsAvailableMemoryWatcher");
return NS_OK;
}
} // namespace mozilla

Просмотреть файл

@ -0,0 +1,56 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_AvailableMemoryWatcherUtils_h
#define mozilla_AvailableMemoryWatcherUtils_h
#include "mozilla/Attributes.h"
#include "nsISupportsUtils.h" // For nsresult
namespace mozilla {
struct MemoryInfo {
unsigned long memTotal;
unsigned long memAvailable;
};
// Check /proc/meminfo for low memory. Largely C method for reading
// /proc/meminfo.
MOZ_MAYBE_UNUSED
static nsresult ReadMemoryFile(const char* meminfoPath, MemoryInfo& aResult) {
FILE* fd;
if ((fd = fopen(meminfoPath, "r")) == nullptr) {
// Meminfo somehow unreachable
return NS_ERROR_FAILURE;
}
char buff[128];
/* The first few lines of meminfo look something like this:
* MemTotal: 65663448 kB
* MemFree: 57368112 kB
* MemAvailable: 61852700 kB
* We mostly care about the available versus the total. We calculate our
* memory thresholds using this, and when memory drops below 5% we consider
* this to be a memory pressure event. In practice these lines aren't
* necessarily in order, but we can simply search for MemTotal
* and MemAvailable.
*/
char namebuffer[20];
while ((fgets(buff, sizeof(buff), fd)) != nullptr) {
if (strstr(buff, "MemTotal:")) {
sscanf(buff, "%s %lu ", namebuffer, &aResult.memTotal);
}
if (strstr(buff, "MemAvailable:")) {
sscanf(buff, "%s %lu ", namebuffer, &aResult.memAvailable);
}
}
fclose(fd);
return NS_OK;
}
} // namespace mozilla
#endif // ifndef mozilla_AvailableMemoryWatcherUtils_h

Просмотреть файл

@ -220,6 +220,13 @@ if CONFIG["OS_TARGET"] == "Darwin":
"MemoryPressureLevelMac.h",
]
if CONFIG["OS_TARGET"] == "Linux":
UNIFIED_SOURCES += [
"AvailableMemoryWatcherLinux.cpp",
]
EXPORTS.mozilla += [
"AvailableMemoryWatcherUtils.h",
]
GeneratedFile("ErrorList.h", script="ErrorList.py", entry_point="error_list_h")
GeneratedFile(

Просмотреть файл

@ -0,0 +1,66 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/AvailableMemoryWatcherUtils.h"
#include <fstream>
#include <iostream>
using namespace mozilla;
const char* kMemInfoPath = "/proc/meminfo";
const char* kTestfilePath = "testdata";
// Test that we are reading some value from /proc/meminfo.
// If the values are nonzero, the test is a success.
void TestFromProc() {
MemoryInfo memInfo{0, 0};
ReadMemoryFile(kMemInfoPath, memInfo);
MOZ_RELEASE_ASSERT(memInfo.memTotal != 0);
MOZ_RELEASE_ASSERT(memInfo.memAvailable != 0);
}
// Test a file using expected syntax.
void TestFromFile() {
MemoryInfo memInfo{0, 0};
std::ofstream aFile(kTestfilePath);
aFile << "MemTotal: 12345 kB\n";
aFile << "MemFree: 99999 kB\n";
aFile << "MemAvailable: 54321 kB\n";
aFile.close();
ReadMemoryFile(kTestfilePath, memInfo);
MOZ_RELEASE_ASSERT(memInfo.memTotal == 12345);
MOZ_RELEASE_ASSERT(memInfo.memAvailable == 54321);
// remove our dummy file
remove(kTestfilePath);
}
// Test a file with useless data. Results should be
// the starting struct with {0,0}.
void TestInvalidFile() {
MemoryInfo memInfo{0, 0};
std::ofstream aFile(kTestfilePath);
aFile << "foo: 12345 kB\n";
aFile << "bar";
aFile.close();
ReadMemoryFile(kTestfilePath, memInfo);
MOZ_RELEASE_ASSERT(memInfo.memTotal == 0);
MOZ_RELEASE_ASSERT(memInfo.memAvailable == 0);
// remove our dummy file
remove(kTestfilePath);
}
int main() {
TestFromProc();
TestFromFile();
TestInvalidFile();
return 0;
}

Просмотреть файл

@ -0,0 +1,227 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include <sys/mman.h> // For memory-locking.
#include "gtest/gtest.h"
#include "AvailableMemoryWatcher.h"
#include "mozilla/Preferences.h"
#include "mozilla/Services.h"
#include "mozilla/SpinEventLoopUntil.h"
#include "mozilla/StaticPrefs_browser.h"
#include "nsIObserverService.h"
#include "nsISupports.h"
#include "nsITimer.h"
#include "nsMemoryPressure.h"
using namespace mozilla;
namespace {
// Dummy tab unloader whose one job is to dispatch a low memory event.
class MockTabUnloader final : public nsITabUnloader {
NS_DECL_THREADSAFE_ISUPPORTS
public:
MockTabUnloader() = default;
NS_IMETHOD UnloadTabAsync() override {
// We want to issue a memory pressure event for
NS_NotifyOfEventualMemoryPressure(MemoryPressureState::LowMemory);
return NS_OK;
}
private:
~MockTabUnloader() = default;
};
NS_IMPL_ISUPPORTS(MockTabUnloader, nsITabUnloader)
// Class that gradually increases the percent memory threshold
// until it reaches 100%, which should guarantee a memory pressure
// notification.
class AvailableMemoryChecker final : public nsITimerCallback, public nsINamed {
public:
NS_DECL_ISUPPORTS
NS_DECL_NSITIMERCALLBACK
NS_DECL_NSINAMED
AvailableMemoryChecker();
void Init();
void Shutdown();
private:
~AvailableMemoryChecker() = default;
bool mResolved;
nsCOMPtr<nsITimer> mTimer;
RefPtr<nsAvailableMemoryWatcherBase> mWatcher;
RefPtr<MockTabUnloader> mTabUnloader;
const uint32_t kPollingInterval = 50;
const uint32_t kPrefIncrement = 5;
};
AvailableMemoryChecker::AvailableMemoryChecker() : mResolved(false) {}
NS_IMPL_ISUPPORTS(AvailableMemoryChecker, nsITimerCallback, nsINamed);
void AvailableMemoryChecker::Init() {
mTabUnloader = new MockTabUnloader;
mWatcher = nsAvailableMemoryWatcherBase::GetSingleton();
mWatcher->RegisterTabUnloader(mTabUnloader);
mTimer = NS_NewTimer();
mTimer->InitWithCallback(this, kPollingInterval,
nsITimer::TYPE_REPEATING_SLACK);
}
void AvailableMemoryChecker::Shutdown() {
if (mTimer) {
mTimer->Cancel();
}
Preferences::ClearUser("browser.low_commit_space_threshold_percent");
}
// Timer callback to increase the pref threshold.
NS_IMETHODIMP
AvailableMemoryChecker::Notify(nsITimer* aTimer) {
uint32_t threshold =
StaticPrefs::browser_low_commit_space_threshold_percent();
if (threshold >= 100) {
mResolved = true;
return NS_OK;
}
threshold += kPrefIncrement;
Preferences::SetUint("browser.low_commit_space_threshold_percent", threshold);
return NS_OK;
}
NS_IMETHODIMP AvailableMemoryChecker::GetName(nsACString& aName) {
aName.AssignLiteral("AvailableMemoryChecker");
return NS_OK;
}
// Class that listens for a given notification, then records
// if it was received.
class Spinner final : public nsIObserver {
nsCOMPtr<nsIObserverService> mObserverSvc;
nsDependentCString mTopic;
bool mTopicObserved;
~Spinner() = default;
public:
NS_DECL_ISUPPORTS
Spinner(nsIObserverService* aObserverSvc, const char* aTopic)
: mObserverSvc(aObserverSvc), mTopic(aTopic), mTopicObserved(false) {}
NS_IMETHOD Observe(nsISupports* aSubject, const char* aTopic,
const char16_t* aData) override {
if (mTopic == aTopic) {
mTopicObserved = true;
mObserverSvc->RemoveObserver(this, aTopic);
// Force the loop to move in case there is no event in the queue.
nsCOMPtr<nsIRunnable> dummyEvent = new Runnable(__func__);
NS_DispatchToMainThread(dummyEvent);
}
return NS_OK;
}
void StartListening() {
mObserverSvc->AddObserver(this, mTopic.get(), false);
}
bool TopicObserved() { return mTopicObserved; }
bool WaitForNotification();
};
NS_IMPL_ISUPPORTS(Spinner, nsIObserver);
bool Spinner::WaitForNotification() {
bool isTimeout = false;
nsCOMPtr<nsITimer> timer;
// This timer should time us out if we never observe our notification.
// Set to 5000 since the memory checker should finish incrementing the
// pref by then, and if it hasn't then it is probably stuck somehow.
NS_NewTimerWithFuncCallback(
getter_AddRefs(timer),
[](nsITimer*, void* isTimeout) {
*reinterpret_cast<bool*>(isTimeout) = true;
},
&isTimeout, 5000, nsITimer::TYPE_ONE_SHOT, __func__);
SpinEventLoopUntil("Spinner:WaitForNotification"_ns, [&]() -> bool {
if (isTimeout) {
return true;
}
return mTopicObserved;
});
return !isTimeout;
}
void StartUserInteraction(const nsCOMPtr<nsIObserverService>& aObserverSvc) {
aObserverSvc->NotifyObservers(nullptr, "user-interaction-active", nullptr);
}
TEST(AvailableMemoryWatcher, BasicTest)
{
nsCOMPtr<nsIObserverService> observerSvc = services::GetObserverService();
RefPtr<Spinner> aSpinner = new Spinner(observerSvc, "memory-pressure");
aSpinner->StartListening();
// Start polling for low memory.
StartUserInteraction(observerSvc);
RefPtr<AvailableMemoryChecker> checker = new AvailableMemoryChecker();
checker->Init();
aSpinner->WaitForNotification();
// The checker should have dispatched a low memory event before reaching 100%
// memory pressure threshold, so the topic should be observed by the spinner.
EXPECT_TRUE(aSpinner->TopicObserved());
checker->Shutdown();
}
TEST(AvailableMemoryWatcher, MemoryLowToHigh)
{
// Setting this pref to 100 ensures we start in a low memory scenario.
Preferences::SetUint("browser.low_commit_space_threshold_percent", 100);
nsCOMPtr<nsIObserverService> observerSvc = services::GetObserverService();
RefPtr<Spinner> lowMemorySpinner =
new Spinner(observerSvc, "memory-pressure");
lowMemorySpinner->StartListening();
StartUserInteraction(observerSvc);
// Start polling for low memory. We should start with low memory when we start
// the checker.
RefPtr<AvailableMemoryChecker> checker = new AvailableMemoryChecker();
checker->Init();
lowMemorySpinner->WaitForNotification();
EXPECT_TRUE(lowMemorySpinner->TopicObserved());
RefPtr<Spinner> highMemorySpinner =
new Spinner(observerSvc, "memory-pressure-stop");
highMemorySpinner->StartListening();
// Now that we are definitely low on memory, let's reset the pref to 0 to
// exit low memory.
Preferences::SetUint("browser.low_commit_space_threshold_percent", 0);
highMemorySpinner->WaitForNotification();
EXPECT_TRUE(highMemorySpinner->TopicObserved());
checker->Shutdown();
}
} // namespace

Просмотреть файл

@ -120,6 +120,11 @@ if CONFIG["OS_TARGET"] == "Darwin":
"TestMacNSURLEscaping.mm",
]
if CONFIG["OS_TARGET"] == "Linux":
UNIFIED_SOURCES += [
"TestAvailableMemoryWatcherLinux.cpp",
]
if (
CONFIG["WRAP_STL_INCLUDES"]
and CONFIG["CC_TYPE"] != "clang-cl"

Просмотреть файл

@ -11,6 +11,13 @@ TEST_DIRS += [
if CONFIG["OS_ARCH"] == "WINNT":
TEST_DIRS += ["windows"]
if CONFIG["OS_TARGET"] == "Linux":
CppUnitTests(
[
"TestMemoryPressureWatcherLinux",
]
)
EXPORTS.testing += [
"TestHarness.h",
]
@ -39,6 +46,7 @@ XPIDL_SOURCES += [
]
LOCAL_INCLUDES += [
"../base",
"../ds",
]