diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile
index 0cd7302..2964bae 100644
--- a/drivers/usb/dwc3/Makefile
+++ b/drivers/usb/dwc3/Makefile
@@ -6,7 +6,7 @@
 
 dwc3-y					:= core.o
 
-dwc3-y					+= gadget.o ep0.o
+obj-$(CONFIG_USB_DWC3_GADGET)		+= gadget.o ep0.o
 
 obj-$(CONFIG_USB_DWC3_OMAP)		+= dwc3-omap.o
 obj-$(CONFIG_USB_DWC3_PHY_OMAP)		+= ti_usb_phy.o