374 lines
13 KiB
C++
374 lines
13 KiB
C++
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
|
|
* vim: set ts=8 sts=4 et sw=4 tw=99:
|
|
*/
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#include <cstdlib>
|
|
|
|
#include "gc/GCInternals.h"
|
|
#include "gc/Memory.h"
|
|
#include "jsapi-tests/tests.h"
|
|
|
|
#if defined(XP_WIN)
|
|
#include "jswin.h"
|
|
#include <psapi.h>
|
|
#elif defined(XP_UNIX)
|
|
#include <algorithm>
|
|
#include <errno.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/resource.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/types.h>
|
|
#include <unistd.h>
|
|
#else
|
|
#error "Memory mapping functions are not defined for your OS."
|
|
#endif
|
|
|
|
BEGIN_TEST(testGCAllocator)
|
|
{
|
|
size_t PageSize = 0;
|
|
#if defined(XP_WIN)
|
|
# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
|
|
SYSTEM_INFO sysinfo;
|
|
GetSystemInfo(&sysinfo);
|
|
PageSize = sysinfo.dwPageSize;
|
|
# else // Various APIs are unavailable. This test is disabled.
|
|
return true;
|
|
# endif
|
|
#elif defined(XP_UNIX)
|
|
PageSize = size_t(sysconf(_SC_PAGESIZE));
|
|
#else
|
|
return true;
|
|
#endif
|
|
|
|
/* Finish any ongoing background free activity. */
|
|
js::gc::FinishGC(cx);
|
|
|
|
bool growUp;
|
|
CHECK(addressesGrowUp(&growUp));
|
|
|
|
if (growUp)
|
|
return testGCAllocatorUp(PageSize);
|
|
return testGCAllocatorDown(PageSize);
|
|
}
|
|
|
|
static const size_t Chunk = 512 * 1024;
|
|
static const size_t Alignment = 2 * Chunk;
|
|
static const int MaxTempChunks = 4096;
|
|
static const size_t StagingSize = 16 * Chunk;
|
|
|
|
bool
|
|
addressesGrowUp(bool* resultOut)
|
|
{
|
|
/*
|
|
* Try to detect whether the OS allocates memory in increasing or decreasing
|
|
* address order by making several allocations and comparing the addresses.
|
|
*/
|
|
|
|
static const unsigned ChunksToTest = 20;
|
|
static const int ThresholdCount = 15;
|
|
|
|
void* chunks[ChunksToTest];
|
|
for (unsigned i = 0; i < ChunksToTest; i++) {
|
|
chunks[i] = mapMemory(2 * Chunk);
|
|
CHECK(chunks[i]);
|
|
}
|
|
|
|
int upCount = 0;
|
|
int downCount = 0;
|
|
|
|
for (unsigned i = 0; i < ChunksToTest - 1; i++) {
|
|
if (chunks[i] < chunks[i + 1])
|
|
upCount++;
|
|
else
|
|
downCount++;
|
|
}
|
|
|
|
for (unsigned i = 0; i < ChunksToTest; i++)
|
|
unmapPages(chunks[i], 2 * Chunk);
|
|
|
|
/* Check results were mostly consistent. */
|
|
CHECK(abs(upCount - downCount) >= ThresholdCount);
|
|
|
|
*resultOut = upCount > downCount;
|
|
|
|
return true;
|
|
}
|
|
|
|
size_t
|
|
offsetFromAligned(void* p)
|
|
{
|
|
return uintptr_t(p) % Alignment;
|
|
}
|
|
|
|
enum AllocType {
|
|
UseNormalAllocator,
|
|
UseLastDitchAllocator
|
|
};
|
|
|
|
bool
|
|
testGCAllocatorUp(const size_t PageSize)
|
|
{
|
|
const size_t UnalignedSize = StagingSize + Alignment - PageSize;
|
|
void* chunkPool[MaxTempChunks];
|
|
// Allocate a contiguous chunk that we can partition for testing.
|
|
void* stagingArea = mapMemory(UnalignedSize);
|
|
if (!stagingArea)
|
|
return false;
|
|
// Ensure that the staging area is aligned.
|
|
unmapPages(stagingArea, UnalignedSize);
|
|
if (offsetFromAligned(stagingArea)) {
|
|
const size_t Offset = offsetFromAligned(stagingArea);
|
|
// Place the area at the lowest aligned address.
|
|
stagingArea = (void*)(uintptr_t(stagingArea) + (Alignment - Offset));
|
|
}
|
|
mapMemoryAt(stagingArea, StagingSize);
|
|
// Make sure there are no available chunks below the staging area.
|
|
int tempChunks;
|
|
if (!fillSpaceBeforeStagingArea(tempChunks, stagingArea, chunkPool, false))
|
|
return false;
|
|
// Unmap the staging area so we can set it up for testing.
|
|
unmapPages(stagingArea, StagingSize);
|
|
// Check that the first chunk is used if it is aligned.
|
|
CHECK(positionIsCorrect("xxooxxx---------", stagingArea, chunkPool, tempChunks));
|
|
// Check that the first chunk is used if it can be aligned.
|
|
CHECK(positionIsCorrect("x-ooxxx---------", stagingArea, chunkPool, tempChunks));
|
|
// Check that an aligned chunk after a single unalignable chunk is used.
|
|
CHECK(positionIsCorrect("x--xooxxx-------", stagingArea, chunkPool, tempChunks));
|
|
// Check that we fall back to the slow path after two unalignable chunks.
|
|
CHECK(positionIsCorrect("x--xx--xoo--xxx-", stagingArea, chunkPool, tempChunks));
|
|
// Check that we also fall back after an unalignable and an alignable chunk.
|
|
CHECK(positionIsCorrect("x--xx---x-oo--x-", stagingArea, chunkPool, tempChunks));
|
|
// Check that the last ditch allocator works as expected.
|
|
CHECK(positionIsCorrect("x--xx--xx-oox---", stagingArea, chunkPool, tempChunks,
|
|
UseLastDitchAllocator));
|
|
|
|
// Clean up.
|
|
while (--tempChunks >= 0)
|
|
unmapPages(chunkPool[tempChunks], 2 * Chunk);
|
|
return true;
|
|
}
|
|
|
|
bool
|
|
testGCAllocatorDown(const size_t PageSize)
|
|
{
|
|
const size_t UnalignedSize = StagingSize + Alignment - PageSize;
|
|
void* chunkPool[MaxTempChunks];
|
|
// Allocate a contiguous chunk that we can partition for testing.
|
|
void* stagingArea = mapMemory(UnalignedSize);
|
|
if (!stagingArea)
|
|
return false;
|
|
// Ensure that the staging area is aligned.
|
|
unmapPages(stagingArea, UnalignedSize);
|
|
if (offsetFromAligned(stagingArea)) {
|
|
void* stagingEnd = (void*)(uintptr_t(stagingArea) + UnalignedSize);
|
|
const size_t Offset = offsetFromAligned(stagingEnd);
|
|
// Place the area at the highest aligned address.
|
|
stagingArea = (void*)(uintptr_t(stagingEnd) - Offset - StagingSize);
|
|
}
|
|
mapMemoryAt(stagingArea, StagingSize);
|
|
// Make sure there are no available chunks above the staging area.
|
|
int tempChunks;
|
|
if (!fillSpaceBeforeStagingArea(tempChunks, stagingArea, chunkPool, true))
|
|
return false;
|
|
// Unmap the staging area so we can set it up for testing.
|
|
unmapPages(stagingArea, StagingSize);
|
|
// Check that the first chunk is used if it is aligned.
|
|
CHECK(positionIsCorrect("---------xxxooxx", stagingArea, chunkPool, tempChunks));
|
|
// Check that the first chunk is used if it can be aligned.
|
|
CHECK(positionIsCorrect("---------xxxoo-x", stagingArea, chunkPool, tempChunks));
|
|
// Check that an aligned chunk after a single unalignable chunk is used.
|
|
CHECK(positionIsCorrect("-------xxxoox--x", stagingArea, chunkPool, tempChunks));
|
|
// Check that we fall back to the slow path after two unalignable chunks.
|
|
CHECK(positionIsCorrect("-xxx--oox--xx--x", stagingArea, chunkPool, tempChunks));
|
|
// Check that we also fall back after an unalignable and an alignable chunk.
|
|
CHECK(positionIsCorrect("-x--oo-x---xx--x", stagingArea, chunkPool, tempChunks));
|
|
// Check that the last ditch allocator works as expected.
|
|
CHECK(positionIsCorrect("---xoo-xx--xx--x", stagingArea, chunkPool, tempChunks,
|
|
UseLastDitchAllocator));
|
|
|
|
// Clean up.
|
|
while (--tempChunks >= 0)
|
|
unmapPages(chunkPool[tempChunks], 2 * Chunk);
|
|
return true;
|
|
}
|
|
|
|
bool
|
|
fillSpaceBeforeStagingArea(int& tempChunks, void* stagingArea,
|
|
void** chunkPool, bool addressesGrowDown)
|
|
{
|
|
// Make sure there are no available chunks before the staging area.
|
|
tempChunks = 0;
|
|
chunkPool[tempChunks++] = mapMemory(2 * Chunk);
|
|
while (tempChunks < MaxTempChunks && chunkPool[tempChunks - 1] &&
|
|
(chunkPool[tempChunks - 1] < stagingArea) ^ addressesGrowDown) {
|
|
chunkPool[tempChunks++] = mapMemory(2 * Chunk);
|
|
if (!chunkPool[tempChunks - 1])
|
|
break; // We already have our staging area, so OOM here is okay.
|
|
if ((chunkPool[tempChunks - 1] < chunkPool[tempChunks - 2]) ^ addressesGrowDown)
|
|
break; // The address growth direction is inconsistent!
|
|
}
|
|
// OOM also means success in this case.
|
|
if (!chunkPool[tempChunks - 1]) {
|
|
--tempChunks;
|
|
return true;
|
|
}
|
|
// Bail if we can't guarantee the right address space layout.
|
|
if ((chunkPool[tempChunks - 1] < stagingArea) ^ addressesGrowDown || (tempChunks > 1 &&
|
|
(chunkPool[tempChunks - 1] < chunkPool[tempChunks - 2]) ^ addressesGrowDown))
|
|
{
|
|
while (--tempChunks >= 0)
|
|
unmapPages(chunkPool[tempChunks], 2 * Chunk);
|
|
unmapPages(stagingArea, StagingSize);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool
|
|
positionIsCorrect(const char* str, void* base, void** chunkPool, int tempChunks,
|
|
AllocType allocator = UseNormalAllocator)
|
|
{
|
|
// str represents a region of memory, with each character representing a
|
|
// region of Chunk bytes. str should contain only x, o and -, where
|
|
// x = mapped by the test to set up the initial conditions,
|
|
// o = mapped by the GC allocator, and
|
|
// - = unmapped.
|
|
// base should point to a region of contiguous free memory
|
|
// large enough to hold strlen(str) chunks of Chunk bytes.
|
|
int len = strlen(str);
|
|
int i;
|
|
// Find the index of the desired address.
|
|
for (i = 0; i < len && str[i] != 'o'; ++i);
|
|
void* desired = (void*)(uintptr_t(base) + i * Chunk);
|
|
// Map the regions indicated by str.
|
|
for (i = 0; i < len; ++i) {
|
|
if (str[i] == 'x')
|
|
mapMemoryAt((void*)(uintptr_t(base) + i * Chunk), Chunk);
|
|
}
|
|
// Allocate using the GC's allocator.
|
|
void* result;
|
|
if (allocator == UseNormalAllocator)
|
|
result = js::gc::MapAlignedPages(2 * Chunk, Alignment);
|
|
else
|
|
result = js::gc::TestMapAlignedPagesLastDitch(2 * Chunk, Alignment);
|
|
// Clean up the mapped regions.
|
|
if (result)
|
|
js::gc::UnmapPages(result, 2 * Chunk);
|
|
for (--i; i >= 0; --i) {
|
|
if (str[i] == 'x')
|
|
js::gc::UnmapPages((void*)(uintptr_t(base) + i * Chunk), Chunk);
|
|
}
|
|
// CHECK returns, so clean up on failure.
|
|
if (result != desired) {
|
|
while (--tempChunks >= 0)
|
|
js::gc::UnmapPages(chunkPool[tempChunks], 2 * Chunk);
|
|
}
|
|
return result == desired;
|
|
}
|
|
|
|
#if defined(XP_WIN)
|
|
# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
|
|
|
|
void*
|
|
mapMemoryAt(void* desired, size_t length)
|
|
{
|
|
return VirtualAlloc(desired, length, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
|
}
|
|
|
|
void*
|
|
mapMemory(size_t length)
|
|
{
|
|
return VirtualAlloc(nullptr, length, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
|
}
|
|
|
|
void
|
|
unmapPages(void* p, size_t size)
|
|
{
|
|
MOZ_ALWAYS_TRUE(VirtualFree(p, 0, MEM_RELEASE));
|
|
}
|
|
|
|
# else // Various APIs are unavailable. This test is disabled.
|
|
|
|
void* mapMemoryAt(void* desired, size_t length) { return nullptr; }
|
|
void* mapMemory(size_t length) { return nullptr; }
|
|
void unmapPages(void* p, size_t size) { }
|
|
|
|
# endif
|
|
#elif defined(XP_UNIX)
|
|
|
|
void*
|
|
mapMemoryAt(void* desired, size_t length)
|
|
{
|
|
#if defined(__ia64__) || (defined(__sparc64__) && defined(__NetBSD__)) || defined(__aarch64__)
|
|
MOZ_RELEASE_ASSERT(0xffff800000000000ULL & (uintptr_t(desired) + length - 1) == 0);
|
|
#endif
|
|
void* region = mmap(desired, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
|
|
if (region == MAP_FAILED)
|
|
return nullptr;
|
|
if (region != desired) {
|
|
if (munmap(region, length))
|
|
MOZ_RELEASE_ASSERT(errno == ENOMEM);
|
|
return nullptr;
|
|
}
|
|
return region;
|
|
}
|
|
|
|
void*
|
|
mapMemory(size_t length)
|
|
{
|
|
int prot = PROT_READ | PROT_WRITE;
|
|
int flags = MAP_PRIVATE | MAP_ANON;
|
|
int fd = -1;
|
|
off_t offset = 0;
|
|
// The test code must be aligned with the implementation in gc/Memory.cpp.
|
|
#if defined(__ia64__) || (defined(__sparc64__) && defined(__NetBSD__))
|
|
void* region = mmap((void*)0x0000070000000000, length, prot, flags, fd, offset);
|
|
if (region == MAP_FAILED)
|
|
return nullptr;
|
|
if ((uintptr_t(region) + (length - 1)) & 0xffff800000000000) {
|
|
if (munmap(region, length))
|
|
MOZ_RELEASE_ASSERT(errno == ENOMEM);
|
|
return nullptr;
|
|
}
|
|
return region;
|
|
#elif defined(__aarch64__)
|
|
const uintptr_t start = UINT64_C(0x0000070000000000);
|
|
const uintptr_t end = UINT64_C(0x0000800000000000);
|
|
const uintptr_t step = js::gc::ChunkSize;
|
|
uintptr_t hint;
|
|
void* region = MAP_FAILED;
|
|
for (hint = start; region == MAP_FAILED && hint + length <= end; hint += step) {
|
|
region = mmap((void*)hint, length, prot, flags, fd, offset);
|
|
if (region != MAP_FAILED) {
|
|
if ((uintptr_t(region) + (length - 1)) & 0xffff800000000000) {
|
|
if (munmap(region, length)) {
|
|
MOZ_RELEASE_ASSERT(errno == ENOMEM);
|
|
}
|
|
region = MAP_FAILED;
|
|
}
|
|
}
|
|
}
|
|
return region == MAP_FAILED ? nullptr : region;
|
|
#else
|
|
void* region = mmap(nullptr, length, prot, flags, fd, offset);
|
|
if (region == MAP_FAILED)
|
|
return nullptr;
|
|
return region;
|
|
#endif
|
|
}
|
|
|
|
void
|
|
unmapPages(void* p, size_t size)
|
|
{
|
|
if (munmap(p, size))
|
|
MOZ_RELEASE_ASSERT(errno == ENOMEM);
|
|
}
|
|
|
|
#else // !defined(XP_WIN) && !defined(XP_UNIX)
|
|
#error "Memory mapping functions are not defined for your OS."
|
|
#endif
|
|
END_TEST(testGCAllocator)
|