diff --git a/drivers/qe/Makefile b/drivers/qe/Makefile
index 45a2fff..18fe9ce 100644
--- a/drivers/qe/Makefile
+++ b/drivers/qe/Makefile
@@ -25,8 +25,9 @@
 LIB	:= $(obj)qe.a
 
 COBJS-$(CONFIG_OF_LIBFDT) += fdt.o
-COBJS	:= qe.o uccf.o uec.o uec_phy.o $(COBJS-y)
+COBJS-$(CONFIG_QE) += qe.o uccf.o uec.o uec_phy.o
 
+COBJS	:= $(COBJS-y)
 SRCS	:= $(COBJS:.o=.c)
 OBJS	:= $(addprefix $(obj),$(COBJS))