With & WhereHas returning empty nested relation

I have the following Eloquent query which to my understanding should return Categories that have relations to attribute sets, which in turn have relations to attributes, which in turn have relationships to variants.

$attributes = Category::with(['attributeSets.attributes.variants' => function ($query) {
            $query->where('id', 1)->withPivot('value');
        }])->whereHas('attributeSets.attributes.variants', function ($query) {
            $query->where('id', 1);
        })->get()->toArray();

However, I receive the following which appears to include attributes with no relation to variants (e.g.) "variants" => [].

How can I remove these attributes?

array:9 [▼
  0 => array:12 [▼
    "id" => 7
    "parent_id" => null
    "category_type_id" => null
    "name" => "harum"
    "identifier" => "harum"
    "slug" => "harum"
    "sort_order" => 0
    "is_active" => 0
    "created_at" => "2019-12-11 15:45:51"
    "updated_at" => "2019-12-11 15:45:51"
    "parent" => null
    "attribute_sets" => array:1 [▼
      0 => array:8 [▼
        "id" => 1
        "name" => "ex"
        "identifier" => "ex"
        "created_at" => "2019-12-11 15:45:51"
        "updated_at" => "2019-12-11 15:45:51"
        "mainCategory" => array:12 [▶]
        "pivot" => array:2 [▶]
        "attributes" => array:3 [▼
          0 => array:8 [▼
            "id" => 1
            "attribute_set_id" => 1
            "name" => "minus"
            "identifier" => "minus"
            "created_at" => "2019-12-11 15:45:51"
            "updated_at" => "2019-12-11 15:45:51"
            "set" => array:6 [▶]
            "variants" => []
          ]
          1 => array:8 [▼
            "id" => 2
            "attribute_set_id" => 1
            "name" => "ducimus"
            "identifier" => "ducimus"
            "created_at" => "2019-12-11 15:45:51"
            "updated_at" => "2019-12-11 15:45:51"
            "set" => array:6 [▶]
            "variants" => []
          ]
          2 => array:8 [▼
            "id" => 3
            "attribute_set_id" => 1
            "name" => "mollitia"
            "identifier" => "mollitia"
            "created_at" => "2019-12-11 15:45:51"
            "updated_at" => "2019-12-11 15:45:51"
            "set" => array:6 [▶]
            "variants" => array:1 [▶]
          ]
        ]
      ]
    ]
  ]

My models:

Category

public function attributeSets()
    {
        return $this->belongsToMany(AttributeSet::class, 'attribute_set_categories');
    }

AttributeSet

public function attributes()
    {
        return $this->hasMany(Attribute::class);
    }

Attribute

public function variants()
    {
        return $this->belongsToMany(Variant::class, 'variant_attributes');
    }

via Chebli Mohamed