diff --git a/gralloc/service.cpp b/gralloc/service.cpp
index 1136e2f7..b19cc884 100644
--- a/gralloc/service.cpp
+++ b/gralloc/service.cpp
@@ -37,7 +37,7 @@ using android::hardware::graphics::allocator::V2_0::IAllocator;
 
 int main(int, char **) {
   android::sp<IAllocator> service = new QtiAllocator();
-  configureRpcThreadpool(1, true /*callerWillJoin*/);
+  configureRpcThreadpool(4, true /*callerWillJoin*/);
   if (service->registerAsService() != android::OK) {
     ALOGE("Cannot register QTI Allocator service");
     return -EINVAL;
diff --git a/sdm/libs/hwc2/hwc_buffer_allocator.cpp b/sdm/libs/hwc2/hwc_buffer_allocator.cpp
index 55988f1e..dab98858 100644
--- a/sdm/libs/hwc2/hwc_buffer_allocator.cpp
+++ b/sdm/libs/hwc2/hwc_buffer_allocator.cpp
@@ -47,7 +47,12 @@ using android::hardware::hidl_vec;
 
 namespace sdm {
 
-DisplayError HWCBufferAllocator::Init() {
+DisplayError HWCBufferAllocator::GetGrallocInstance() {
+  // Lazy initialization of gralloc HALs
+  if (mapper_ != nullptr || allocator_ != nullptr) {
+    return kErrorNone;
+  }
+
   allocator_ = IAllocator::getService();
   mapper_ = IMapper::getService();
 
@@ -58,11 +63,11 @@ DisplayError HWCBufferAllocator::Init() {
   return kErrorNone;
 }
 
-DisplayError HWCBufferAllocator::Deinit() {
-  return kErrorNone;
-}
-
 DisplayError HWCBufferAllocator::AllocateBuffer(BufferInfo *buffer_info) {
+  auto err = GetGrallocInstance();
+  if (err != kErrorNone) {
+    return err;
+  }
   const BufferConfig &buffer_config = buffer_info->buffer_config;
   AllocatedBufferInfo *alloc_buffer_info = &buffer_info->alloc_buffer_info;
   int format;
diff --git a/sdm/libs/hwc2/hwc_buffer_allocator.h b/sdm/libs/hwc2/hwc_buffer_allocator.h
index 2846b893..d6f251f7 100644
--- a/sdm/libs/hwc2/hwc_buffer_allocator.h
+++ b/sdm/libs/hwc2/hwc_buffer_allocator.h
@@ -48,8 +48,6 @@ inline Type ALIGN(Type x, Type align) {
 
 class HWCBufferAllocator : public BufferAllocator {
  public:
-  DisplayError Init();
-  DisplayError Deinit();
   DisplayError AllocateBuffer(BufferInfo *buffer_info);
   DisplayError FreeBuffer(BufferInfo *buffer_info);
   uint32_t GetBufferSize(BufferInfo *buffer_info);
@@ -66,6 +64,7 @@ class HWCBufferAllocator : public BufferAllocator {
   DisplayError UnmapBuffer(const private_handle_t *handle, int *release_fence);
 
  private:
+  DisplayError GetGrallocInstance();
   android::sp<IMapper> mapper_;
   android::sp<IAllocator> allocator_;
 };
diff --git a/sdm/libs/hwc2/hwc_session.cpp b/sdm/libs/hwc2/hwc_session.cpp
index 02e2129e..74ffe382 100644
--- a/sdm/libs/hwc2/hwc_session.cpp
+++ b/sdm/libs/hwc2/hwc_session.cpp
@@ -160,23 +160,10 @@ int HWCSession::Init() {
 
   StartServices();
 
-  DisplayError error = buffer_allocator_.Init();
-  if (error != kErrorNone) {
-    ALOGE("%s::%s: Buffer allocator initialization failed. Error = %d",
-          __CLASS__, __FUNCTION__, error);
-    return -EINVAL;
-  }
-
   g_hwc_uevent_.Register(this);
 
-  error = CoreInterface::CreateCore(HWCDebugHandler::Get(), &buffer_allocator_,
-                                    &buffer_sync_handler_, &socket_handler_, &core_intf_);
-  if (error != kErrorNone) {
-    buffer_allocator_.Deinit();
-    ALOGE("%s::%s: Display core initialization failed. Error = %d", __CLASS__, __FUNCTION__, error);
-    return -EINVAL;
-  }
-
+  auto error = CoreInterface::CreateCore(HWCDebugHandler::Get(), &buffer_allocator_,
+                                         &buffer_sync_handler_, &socket_handler_, &core_intf_);
 
   // If HDMI display is primary display, defer display creation until hotplug event is received.
   HWDisplayInterfaceInfo hw_disp_info = {};
@@ -184,7 +171,6 @@ int HWCSession::Init() {
   if (error != kErrorNone) {
     g_hwc_uevent_.Register(nullptr);
     CoreInterface::DestroyCore();
-    buffer_allocator_.Deinit();
     DLOGE("Primary display type not recognized. Error = %d", error);
     return -EINVAL;
   }
@@ -209,7 +195,6 @@ int HWCSession::Init() {
   if (status) {
     g_hwc_uevent_.Register(nullptr);
     CoreInterface::DestroyCore();
-    buffer_allocator_.Deinit();
     return status;
   }